From 9a2730d66e32b5e6f0e63238b270f5c7ebd19edb Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Sun, 20 Nov 2016 16:52:44 -0500 Subject: [PATCH 01/11] WIP - everything is broken --- database-private-beets.js | 0 modules/configure-commander.js | 1 - modules/configure-json-file.js | 8 +- modules/database-beets.js | 12 +- modules/database-default.js | 46 +++-- modules/database-master.js | 54 ++++++ modules/database-private-mstream.js | 5 + modules/database-public-sqlite.js | 36 ++++ mstream.js | 220 ++++++++++++++++------- public/js/mstream.js | 9 +- untitled folder/exampleNewJSON.json | 40 +++++ untitled folder/new-db-model.txt | 60 +++++++ untitled folder/privatePublicDBModel.txt | 60 +++++++ 13 files changed, 468 insertions(+), 83 deletions(-) create mode 100644 database-private-beets.js create mode 100644 modules/database-master.js create mode 100644 modules/database-private-mstream.js create mode 100644 modules/database-public-sqlite.js create mode 100644 untitled folder/exampleNewJSON.json create mode 100644 untitled folder/new-db-model.txt create mode 100644 untitled folder/privatePublicDBModel.txt diff --git a/database-private-beets.js b/database-private-beets.js new file mode 100644 index 0000000..e69de29 diff --git a/modules/configure-commander.js b/modules/configure-commander.js index 39af183..1373df9 100644 --- a/modules/configure-commander.js +++ b/modules/configure-commander.js @@ -7,7 +7,6 @@ exports.setup = function(args){ .option('-p, --port ', 'Select Port', /^\d+$/i, 3000) .option('-t, --tunnel', 'Use nat-pmp to configure port fowarding') .option('-g, --gateip ', 'Manually set gateway IP for the tunnel option') - .option('-l, --login', 'Require users to login') .option('-u, --user ', 'Set Username') .option('-x, --password ', 'Set Password') .option('-e, --email ', 'Set User Email (optional)') diff --git a/modules/configure-json-file.js b/modules/configure-json-file.js index 1f13f37..3f5954d 100644 --- a/modules/configure-json-file.js +++ b/modules/configure-json-file.js @@ -11,8 +11,8 @@ exports.setup = function(args){ } // Check for validity - if(!loadJson.filepath){ - loadJson.filepath = process.cwd(); + if(!loadJson.user){ + loadJson.user = 'admin'; } if(!loadJson.databaseplugin){ @@ -25,6 +25,10 @@ exports.setup = function(args){ } } + if(!loadJson.filepath){ + loadJson.filepath = process.cwd(); + } + // Export JSON return loadJson; } diff --git a/modules/database-beets.js b/modules/database-beets.js index c0eb1c7..2a53ce3 100644 --- a/modules/database-beets.js +++ b/modules/database-beets.js @@ -5,10 +5,18 @@ var scanLock = false; var yetAnotherArrayOfSongs = []; var totalFileCount = 0; -exports.setup = function(mstream, program, rootDir, db){ - const scanThisDir = program.beetspath; // TODO: Check that this is a real directory +function runOnStart(){ + // Create a playlist table + db.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, user varchar, created datetime default current_timestamp);", function() { + // console.log('PLAYLIST TABLE CREATED'); + }); +} +exports.setup = function(mstream, users, db){ // TODO + // const scanThisDir = program.beetspath; // TODO: Check that this is a real directory + // TODO: pull scanThisDir from the users array + mstream.get('/db/recursive-scan', function(req,res){ if(scanLock === true){ diff --git a/modules/database-default.js b/modules/database-default.js index 7077667..1330195 100644 --- a/modules/database-default.js +++ b/modules/database-default.js @@ -1,6 +1,8 @@ -const metadata = require('musicmetadata'); // TODO: Look into replacing with taglib +const metadata = require('musicmetadata'); const fs = require('graceful-fs'); // File System const fe = require('path'); +const crypto = require('crypto'); + var dbCopy; @@ -9,6 +11,12 @@ var scanLock = false; var yetAnotherArrayOfSongs = []; var totalFileCount = 0; +//TODO: Spawn new thread for processing files +// Handle users when processing files +// Hash files when processing +// Handle album art when processing files +// Use created/modified dates to handle updating DB + function getFileType(filename){ return filename.split(".").pop(); @@ -151,17 +159,35 @@ function countFiles (dir, fileTypesArray) { } +function runOnStart(){ + // If we are not using Beets DB, we need to prep the DB + dbCopy.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { + // console.log('TABLES CREATED'); + }); + + // Create a playlist table + dbCopy.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, user varchar, created datetime default current_timestamp);", function() { + // console.log('PLAYLIST TABLE CREATED'); + }); + + // Loop through users + + // Scan once at a time -exports.setup = function(mstream, program, rootDir, db){ +} + + + +exports.setup = function(mstream, users, db){ const rootDirCopy = rootDir; dbCopy = db; + runOnStart(); + // scan and screate database mstream.get('/db/recursive-scan', function(req,res){ - console.log('xxx'); - // Check if this is already running if(scanLock === true){ // Return error @@ -173,21 +199,12 @@ exports.setup = function(mstream, program, rootDir, db){ // turn on scan lock scanLock = true; - // Make sure directory exits var fileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; - - console.log(rootDirCopy); - countFiles(rootDirCopy, fileTypesArray); - - totalFileCount = yetAnotherArrayOfSongs.length; - console.log(totalFileCount); - - dbCopy.serialize(function() { // These two queries will run sequentially. dbCopy.run("drop table if exists items;"); @@ -205,8 +222,7 @@ exports.setup = function(mstream, program, rootDir, db){ // Remove lock scanLock = false; - // // Log error - // res.status(500).send('{"error":"'+err+'"}'); + // TODO Log error // console.log(err); return; } diff --git a/modules/database-master.js b/modules/database-master.js new file mode 100644 index 0000000..a1f9b7c --- /dev/null +++ b/modules/database-master.js @@ -0,0 +1,54 @@ +exports.setup = function(mstream, users, publicDBType){ + // Go through all vars and determine plugins needed + var privateDB = false; + for (i = 0; i < program.users.length; i++) { + // Check for beets + // privateDB = sqlite3; + } + // sqlite3, mysql, sequelize, lokiJS, (? posgres) + // This will allow us to make sqlite3 an optional dependancy once lokiJS works + // Make sure all users have uniform settings for the inital run, we'll handle mixed settings later + + + + + + // Load the public DB plugin + // sqlite3 and mysql to start, add lokiJS support latet (and maybe posgres later), maybe a NO DB option even later +// The following api calls are all handled on a public level +// They can be moved into a public plugin +// pull plugin from masterDBType +const mstreamPublicDB = require('./modules/database-public-'+publicDBType+'.js'); +mstreamPublicDB.setup(mstream); + + + + + +// TODO: Load any plugins necessary for habdling indivudal user dbs + // Then construct routing between api calls and userDB management functions + + // TODO: Handle DB write functions + mstream.get('/db/status', function(req, res){ + }); + mstream.get('/db/recursive-scan', function(req,res){ + }); + + // TODO: Handle Specialized DB Functions + // mstream.get('/db/download-db', function(req, res){ + // }); + // mstream.get( '/db/hash', function(req, res){ + // }); + +} + + + +// Case 1: totalally managed by mstream, sqlite3 for everything + +// Case 2: beets DB for every user, mysql for public DB + +// Case 3: totally managed by mstream, lokiJS for everything + +// Next step: mixed user settings +// Next step: backup as local DB and import from backup diff --git a/modules/database-private-mstream.js b/modules/database-private-mstream.js new file mode 100644 index 0000000..49c9326 --- /dev/null +++ b/modules/database-private-mstream.js @@ -0,0 +1,5 @@ +// Function that scans user folder and returns array of all new songs to process + // Make this a generator so it can work in pieces + +// Function that can spawn new process for scannign user folders + // Queue scans so we don't start too many at a time diff --git a/modules/database-public-sqlite.js b/modules/database-public-sqlite.js new file mode 100644 index 0000000..5bc1f8f --- /dev/null +++ b/modules/database-public-sqlite.js @@ -0,0 +1,36 @@ +// function that takes in a json array of songs and saves them to the sqlite db + // must contain the username and filepath for each song + +// function that gets artist info and returns json array of albums +// function that searches db and returns json array of albums and artists +// function that takes ina playlsit name and searchs db for that playlist and returns a json array of songs for that playlist +// BASICALLY, all the functions we have no but de-couple them from the Express API calls + + +exports.setup = function(mstream){ + // Attach API calls to functions + mstream.get('/getallplaylists', function (req, res){ + // Check user db settings + // pull function name out of user settings + // launch funtion + // Example functions: saveViamysql, saveViaSqlite, saveViaLoki, saveToFile + }); + mstream.get('/loadplaylist', function (req, res){ + }); + mstream.get('/deleteplaylist', function(req, res){ + }); + + + mstream.post('/db/search', function(req, res){ + }); + mstream.get('/db/artists', function (req, res) { + }); + mstream.post('/db/artists-albums', function (req, res) { + }); + mstream.get('/db/albums', function (req, res) { + }); + mstream.post('/db/album-songs', function (req, res) { + }); +//================================================================================ + +} diff --git a/mstream.js b/mstream.js index dab5260..38f97a2 100755 --- a/mstream.js +++ b/mstream.js @@ -10,7 +10,7 @@ const archiver = require('archiver'); // Zip Compression const os = require('os'); const crypto = require('crypto'); const slash = require('slash'); -const sqlite3 = require('sqlite3').verbose(); +// const sqlite3 = require('sqlite3').verbose(); var startup = 'configure-commander'; @@ -28,35 +28,68 @@ const program = require('./modules/' + startup + '.js').setup(process.argv); if(program == false){ process.exit(); } -const db = new sqlite3.Database(program.database); +// const db = new sqlite3.Database(program.database); -// If we are not using Beets DB, we need to prep the DB -if(program.databaseplugin === 'default'){ - db.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { - // console.log('TABLES CREATED'); - }); -} -// Create a playlist table -db.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, created datetime default current_timestamp);", function() { - // console.log('PLAYLIST TABLE CREATED'); -}); +// TODO: Move these to db modules +// // If we are not using Beets DB, we need to prep the DB +// if(program.databaseplugin === 'default'){ +// db.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { +// // console.log('TABLES CREATED'); +// }); +// } +// // Create a playlist table +// db.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, created datetime default current_timestamp);", function() { +// // console.log('PLAYLIST TABLE CREATED'); +// }); // Normalize for all OS // Make sure it's a directory -if(!fs.statSync( program.filepath ).isDirectory()){ +// Loop through and makeure all user Dirs are real +if(program.users){ + for (i = 0; i < program.users.length; i++) { + //TODO: Check usernames for forbidden chars + + // TODO: Assure all usernames are unique + // TODO: Or update JSON so usernames have to be unique + + // TODO: Assure only one user per filepath + + // TODO: Assure all users are usingthe same DB schema + + if(!fs.statSync( program.users[i].musicDir ).isDirectory()){ + console.log(program.users[i].username + " music directory could not be found"); + process.exit(); + } + } +}else if(!fs.statSync( program.filepath ).isDirectory()){ console.log('GIVEN DIRECTORY DOES NOT APPEAR TO BE REAL'); process.exit(); } -const rootDir = fe.normalize(program.filepath); -// Normalize It -if(!fe.isAbsolute(program.filepath) ){ - rootDir = fe.join(process.cwd, rootDir); + +if(program.user && program.password){ + // Move program.username and program.password to program.users + var newUser = { + "username":program.user, + "password":program.password, + "musicDir":program.filepath + }; + + if(program.email){ + newUser.email = program.email + } + + // TODO: Handle Guest Account + // if(program.guest && program.guestPassword){ + + // } + + program.users.push(newUser); } @@ -67,8 +100,22 @@ if(!fs.statSync( fe.join(__dirname, program.userinterface) ).isDirectory()){ } // Static files +// TODO: Loop through and create sperate virtual paths for all user dirs mstream.use( express.static(fe.join(__dirname, program.userinterface) )); -mstream.use( '/' , express.static( rootDir )); +if(program.users){ + for (i = 0; i < program.users.length; i++) { + // TODO: Check if musicDir is real + + mstream.use( '/' + program.users[i].username + '/' , express.static( program.users[i].musicDir )); + } +}else{ + var rootDir = fe.normalize(program.filepath); + // Normalize It + if(!fe.isAbsolute(program.filepath) ){ + rootDir = fe.join(process.cwd, rootDir); + } + mstream.use( '/' , express.static( rootDir )); +} // Magic Middleware Things mstream.use(bodyParser.json()); // support json encoded bodies @@ -102,24 +149,28 @@ mstream.get('/', function (req, res) { // Login functionality -if(program.login){ - if(!program.password || !program.user){ - console.log('User credentials are missing. Please make sure to supply both a username and password via the -u and -p commands respectivly. Aborting'); - process.exit(1); - } +if(program.users){ // TODO: password change function if(program.email){ - mstream.get('/change-password-request', function (req, res) { + mstream.post('/change-password-request', function (req, res) { + // Get email address from request + // validate email against user array + // Generate change password token // Invalidate all other change password tokens // Email the user the token - + res.sendFile( 'COMING SOON!' ); }); + + // TODO: Add New user + // Check for root user and password + // Add credentials to user array + mstream.post('/change-password', function (req, res){ // Check token @@ -159,11 +210,26 @@ if(program.login){ // Create the user array var Users = {}; - Users[program.user] = { - 'guest': false, - 'password':'', - } + // TODO: Construct user array + for (i = 0; i < program.users.length; i++) { + Users[program.users[i].username] = { + "musicDir":program.users[i].musicDir, + } + if(program.users[i].email){ + Users[program.users[i].username].email = program.users[i].email; + } + } + // Users[program.user] = { + // "guest": false, + // "guestPassword":"", + // "password":'', + // "email":"", + // "musicDir":"", + // + // } + + // TODO: Break salt generation and password management into a loop and seperate function // Encrypt the password bcrypt.genSalt(10, function(err, salt) { bcrypt.hash(program.password, salt, function(err, hash) { @@ -171,14 +237,12 @@ if(program.login){ Users[program.user]['password'] = hash; }); }); - // Handle guest account if(program.guest && program.guestpassword){ Users[program.guest] = { 'guest': true, 'password':'', } - // Encrypt the password bcrypt.genSalt(10, function(err, salt) { bcrypt.hash(program.guestpassword, salt, function(err, hash) { @@ -188,6 +252,8 @@ if(program.login){ }); } + + // Failed Login Attempt mstream.get('/login-failed', function (req, res) { // Wait before sending the response @@ -259,25 +325,44 @@ if(program.login){ } // Deny guest access - req.decoded = decoded; if(decoded.guest === true && forbiddenFunctions.indexOf(req.path) != -1){ return res.redirect('/guest-access-denied'); } + // Set user request data + req.user = decoded; + + next(); }); }); // TODO: Authenticate all HTTP requests for music files (mp3 and other formats) +}else{ + + // Dummy data + mstream.use(function(req, res, next) { + req.user = { + "username":"", + "musicDir":program.filepath + }; + next(); + }); + } // Test function // Used to determine the user has a working login token +// TODO: This will return the virtual file path directory needed to access msuci files mstream.get('/ping', function(req, res){ - res.send('pong'); + var returnObject = { + 'vPath' = req.user.username, + 'guest' = false, // TODO: return guest status + }; + res.send(JSON.stringify(returnObject); }); @@ -288,12 +373,13 @@ mstream.post('/dirparser', function (req, res) { var filesArray = []; // Make sure directory exits - var path = req.body.dir; - if(path == ""){ - path = rootDir; - }else{ - path = fe.join(rootDir, path); - } + // TODO Get music dir from request + var path = fe.join(req.user.musicDir, req.body.dir); + // if(path == ""){ + // path = rootDir; + // }else{ + // path = fe.join(rootDir, path); + // } // Will only show these files. Prevents people from snooping around // TODO: Move to global vairable @@ -355,8 +441,8 @@ mstream.post('/dirparser', function (req, res) { } - - var returnPath = slash( fe.relative(rootDir, path) ); + // TODO: rootdir stuff here + var returnPath = slash( fe.relative(req.user.musicDir, path) ); if(returnPath.slice(-1) !== '/'){ returnPath += '/'; @@ -376,12 +462,13 @@ function getFileType(filename){ - +// TODO: Save playlist according to user and the user's music DIR mstream.post('/saveplaylist', function (req, res){ var title = req.body.title; var songs = req.body.stuff; // Check if this playlist already exists + // TODO: Add field for username db.all("SELECT id FROM mstream_playlists WHERE playlist_name = ?;", title, function(err, rows) { db.serialize(function() { @@ -400,7 +487,7 @@ mstream.post('/saveplaylist', function (req, res){ sql2 += "(?, ?), "; sqlParser.push(title); - sqlParser.push( fe.join(rootDir, song) ); + sqlParser.push( fe.join(req.user.musicDir, song) ); // TODO: User music dir } sql2 = sql2.slice(0, -2); @@ -444,7 +531,7 @@ mstream.get('/loadplaylist', function (req, res){ // var tempName = rows[i].filepath.split('/').slice(-1)[0]; var tempName = fe.basename(rows[i].filepath); var extension = getFileType(rows[i].filepath); - var filepath = slash(fe.relative(rootDir, rows[i].filepath)); + var filepath = slash(fe.relative(req.user.musicDir, rows[i].filepath)); // TODO returnThis.push({name: tempName, file: filepath, filetype: extension }); } @@ -521,10 +608,14 @@ mstream.post('/download', function (req, res){ }); +// Old way +//const mstreamDB = require('./modules/database-'+program.databaseplugin+'.js'); +// mstreamDB.setup(mstream, program.users, db); // TODO: ROOTDIR -const mstreamDB = require('./modules/database-'+program.databaseplugin+'.js'); -mstreamDB.setup(mstream, program, rootDir, db); - +// New Way +var publicDBType = 'sqlite3'; // Can be sqlite3/mysql/LokiJS +const mstreamDB = require('./modules/database-master.js'); +mstreamDB.setup(mstream, program.users, publicDBType); mstream.post('/db/search', function(req, res){ @@ -670,7 +761,14 @@ mstream.post('/db/album-songs', function (req, res) { } // Format data for API - rows = setLocalFileLocation(rows); + // rows = setLocalFileLocation(rows); + for(var i in rows ){ + var path = String(rows[i]['cast(path as TEXT)']); + + rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase + rows[i].file_location = slash(fe.relative(req.user.musicDir, path)); // Get the local file location + rows[i].filename = fe.basename( path ); // Ge the filname + } res.send(JSON.stringify(rows)); @@ -679,19 +777,19 @@ mstream.post('/db/album-songs', function (req, res) { - -function setLocalFileLocation(rows){ - - for(var i in rows ){ - var path = String(rows[i]['cast(path as TEXT)']); - - rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase - rows[i].file_location = slash(fe.relative(rootDir, path)); // Get the local file location - rows[i].filename = fe.basename( path ); // Ge the filname - } - - return rows; -} +// // TODO +// function setLocalFileLocation(rows){ +// +// for(var i in rows ){ +// var path = String(rows[i]['cast(path as TEXT)']); +// +// rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase +// rows[i].file_location = slash(fe.relative(rootDir, path)); // Get the local file location +// rows[i].filename = fe.basename( path ); // Ge the filname +// } +// +// return rows; +// } diff --git a/public/js/mstream.js b/public/js/mstream.js index 2c88f96..25deb3c 100755 --- a/public/js/mstream.js +++ b/public/js/mstream.js @@ -57,7 +57,6 @@ $(document).ready(function(){ var accessKey = ''; - $.ajaxPrefilter(function( options ) { options.beforeSend = function (xhr) { xhr.setRequestHeader('x-access-token', accessKey); @@ -67,7 +66,7 @@ $(document).ready(function(){ - // Determine if the user needs to log sin + // Determine if the user needs to log in function testIt(){ var token = Cookies.get('token'); if(token){ @@ -95,6 +94,12 @@ $(document).ready(function(){ testIt(); + + // TODO: This var nees to be appened to the beginning of any music fileapath + // This var will either be the username or the value returned by the ping API call + var vPath = ''; + + ////////////////////////////// Initialization code // Supported file types diff --git a/untitled folder/exampleNewJSON.json b/untitled folder/exampleNewJSON.json new file mode 100644 index 0000000..865129d --- /dev/null +++ b/untitled folder/exampleNewJSON.json @@ -0,0 +1,40 @@ +{ + "port":3031, + "login":true, + "users":[ + { + "username":"paul", + "password":"glassjar98", + "email":"paul@wall.com", + "musicDir":"/path/to/music", + "guest":true, + "guestPassword":"abcd", + "bannedGuestFunctions":"todo. make this optional", + "importDB":"path/to/sqlite3.db", + "beetspath":"/path/to/beets/music/dir" + }, + { + "username":"paul2", + "password":"glassjar98", + "email":"paul@wall2.com", + "musicDir":"/path/to/music2", + "guest":"false", + "importDB":"path/to/sqlite3.db" + } + ], + "userinterface":"public", + + "database_public-DEFAULT EXAMPLE SSQLITE3":{ + "type":"sqlite", + "db":"/path/to/db", + }, + "database_public-MYSQL":{ + "type":"mysql", + "username":"lol", + "password":"lol", + "db-name":"lolDB" + }, + "database-public-LOKI":{ + "comming":"soon", + } +} diff --git a/untitled folder/new-db-model.txt b/untitled folder/new-db-model.txt new file mode 100644 index 0000000..9b4193d --- /dev/null +++ b/untitled folder/new-db-model.txt @@ -0,0 +1,60 @@ +UUSER BASED SETINGS +beets-management : true/false +beets-import-path: path where beets gets it's music from +local-user-db : db to port music in from +single-user-backup : true / false + Backs up users files to a private db. Will auto to false if user is using beets + + +GLOBAL SETTINGS +public-db-type: sqlite3, mysql, other + +SHOULD WE MAKE BEETS MANAGEMENT A GLOBAL THING??? + + + + + + +========================================================================================== +The options are: + +build and manage everything on a public scale and ignore all private DBs + This is how mStream Express will do things + +Build everything on a public scale and backup on a private scale (when requested) + // This might be able to be ignored for the first revision + +TODO: Mix beets and no-beets users + +Build eveything on a private scale using beets, and then build a public DB based on that + + + +========================================================================================== + + +- Public DB management plugin system + - input: db type - sqlite, mysql, etc + - input: users array + - input: mstream variable + - loads the appropriate db plugin based on input + - db plugins contain all API calls that construct the DB (playlists, db search, albums, artits, etc) + - (OPTIONAL: Include a NO DB option that saves playlists as files and nothing else) + - Handles all individual user db settings (3 options) + - Beets management + - mstream management - no private DB + - mstream management - private db backup + - holds functions that can manage beets, scan beets for updates and update local DB, and make backups in different db types + - routes user's to appropriate functions + - need to separate the api endpoint functions, the db functions, and the routing that connects the two + - can optionally not load some functions if no users are using them + + + +======================================================================================================== + +NOTES +- all things are going to be based on usernames. + - Should make functions that can generate and save json. + - This way we can have a command that adds and saves a new user diff --git a/untitled folder/privatePublicDBModel.txt b/untitled folder/privatePublicDBModel.txt new file mode 100644 index 0000000..716c6d6 --- /dev/null +++ b/untitled folder/privatePublicDBModel.txt @@ -0,0 +1,60 @@ +Multi user + +make all user music folders available seperatly + +Make one DB that stores everything + Make seperate libraries that can import from different DBs + + +Need a seperate table that logs update times + + +==================================================================================== +Beets DB + Create and store hash of each private DB, and a last updated timestamp + + Compare DB hashes + If different, scan private DB for fields added after timestamp + Add all new fields to public DB + + +Built In DB + Spawn new process + Process all newly added files + Add new files to both private user DB and public shared DB + + + +================================================================================ +The Private/Public DB model + +Each user stores a private sqlite3DB (or maybe something else) +When a filescan is done, both the private DB is updated +An mstream plugin can read the private DB and insert it into a shared public DB + public DBs can easily be created and destroyed to change expanding server needs + + + +HOSTING +Private DBs are mamaged by beets +Public DBs are managed by MySQL + +MSTREAM EXPRESS +NO Private DBS +All users files are managed in one big public DB + + + +==================================================================================== +All Scenarios: + +User is using mstream for everything +sqlite for public and private db + +User is is using sqlite3 for public DB, and beets for private database + +Mysql for public DB, beets for private db + +Mysql for public and private + +Bonus: Custom javascript DB (ex: http://lokijs.org/#/) for everything From 24151dbe9aa541e4aa8c5bcb0a512ac29d08bb2a Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Sun, 27 Nov 2016 01:28:04 -0500 Subject: [PATCH 02/11] More progress. Nothing works yet --- modules/database-beets.js | 119 ---- modules/database-default.js | 271 --------- modules/database-master.js | 54 -- modules/database-private-mstream.js | 5 - modules/database-public-sqlite.js | 36 -- .../db-management/database-beets-manager.js | 115 ++++ .../db-management/database-default-manager.js | 574 ++++++++++++++++++ modules/db-management/database-master.js | 89 +++ modules/db-read/database-public-sqlite.js | 263 ++++++++ modules/db-write/database-beets-mysql.js | 1 + modules/db-write/database-beets-sqlite.js | 1 + modules/db-write/database-default-mysql.js | 2 + modules/db-write/database-default-sqlite.js | 82 +++ mstream.js | 247 ++++---- 14 files changed, 1249 insertions(+), 610 deletions(-) delete mode 100644 modules/database-beets.js delete mode 100644 modules/database-default.js delete mode 100644 modules/database-master.js delete mode 100644 modules/database-private-mstream.js delete mode 100644 modules/database-public-sqlite.js create mode 100644 modules/db-management/database-beets-manager.js create mode 100644 modules/db-management/database-default-manager.js create mode 100644 modules/db-management/database-master.js create mode 100644 modules/db-read/database-public-sqlite.js create mode 100644 modules/db-write/database-beets-mysql.js create mode 100644 modules/db-write/database-beets-sqlite.js create mode 100644 modules/db-write/database-default-mysql.js create mode 100644 modules/db-write/database-default-sqlite.js diff --git a/modules/database-beets.js b/modules/database-beets.js deleted file mode 100644 index 2a53ce3..0000000 --- a/modules/database-beets.js +++ /dev/null @@ -1,119 +0,0 @@ -// TODO: This thing has to be tested - -const spawn = require('child_process').spawn; -var scanLock = false; -var yetAnotherArrayOfSongs = []; -var totalFileCount = 0; - -function runOnStart(){ - // Create a playlist table - db.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, user varchar, created datetime default current_timestamp);", function() { - // console.log('PLAYLIST TABLE CREATED'); - }); -} - - -exports.setup = function(mstream, users, db){ // TODO - // const scanThisDir = program.beetspath; // TODO: Check that this is a real directory - // TODO: pull scanThisDir from the users array - - mstream.get('/db/recursive-scan', function(req,res){ - - if(scanLock === true){ - // Return error - res.status(401).send('{"error":"Scan in progress"}'); - return; - } - - scanLock = true; - var cmd = spawn('beet', [ 'import', '-A', '--group-albums' , scanThisDir]); - - cmd.stdout.on('data', (data) => { - console.log(`stdout: ${data}`); - }); - - cmd.stderr.on('data', (data) => { - console.log(`stderr: ${data}`); - scanLock = false; - - }); - - cmd.on('close', (code) => { - console.log(`child process exited with code ${code}`); - hashFileBeets(); - - // TODO: Remove all empty dirs - }); - }); - - - function hashFileBeets(){ - // var hashCmd = spawn('beet check -a'); - var hashCmd = spawn('beet', [ 'check', '-a']); - - - hashCmd.stdout.on('data', (data) => { - console.log(`stdout: ${data}`); - }); - - hashCmd.stderr.on('data', (data) => { - console.log(`stderr: ${data}`); - scanLock = false; - - }); - - hashCmd.on('close', (code) => { - console.log(`child process exited with code ${code}`); - scanLock = false; - - }); - } - - // TODO: Function that will remove all empty folders - function removeEmptyFolders(){ - var hashCmd = spawn('beet', [ 'check', '-a']); - // 'find ~ -type d -empty -delete' - } - - - - mstream.get('/db/status', function(req, res){ - var returnObject = {}; - - returnObject.locked = scanLock; - - - if(scanLock){ - - // Currently we don't support filecount stats when using beets DB - // Dummy data - returnObject.totalFileCount = 0; - returnObject.filesLeft = 0; - - - res.json(returnObject); - - }else{ - var sql = 'SELECT Count(*) FROM items'; - - db.get(sql, function(err, row){ - if(err){ - console.log(err.message); - - res.status(500).json({ error: err.message }); - return; - } - - - var fileCountDB = row['Count(*)']; // TODO: Is this correct??? - - returnObject.totalFileCount = fileCountDB; - res.json(returnObject); - - }); - } - - }); - - -} diff --git a/modules/database-default.js b/modules/database-default.js deleted file mode 100644 index 1330195..0000000 --- a/modules/database-default.js +++ /dev/null @@ -1,271 +0,0 @@ -const metadata = require('musicmetadata'); -const fs = require('graceful-fs'); // File System -const fe = require('path'); -const crypto = require('crypto'); - -var dbCopy; - - -var arrayOfSongs = []; -var scanLock = false; -var yetAnotherArrayOfSongs = []; -var totalFileCount = 0; - -//TODO: Spawn new thread for processing files -// Handle users when processing files -// Hash files when processing -// Handle album art when processing files -// Use created/modified dates to handle updating DB - - -function getFileType(filename){ - return filename.split(".").pop(); -} - -function parseFile(thisSong){ - var readableStream = fs.createReadStream(thisSong); - var parser = metadata(readableStream, function (err, songInfo) { - if(err){ - // TODO: Do something - } - - - // TODO: Hash the file here and add the hash to the DB - - console.log(songInfo); - - - // Close the stream - readableStream.close(); - - - songInfo.filePath = thisSong; - songInfo.format = getFileType(thisSong); - - arrayOfSongs.push(songInfo); - - - // if there are more than 100 entries, or if it's the last song - if(arrayOfSongs.length > 99){ - insertEntries(); - } - - // For the generator - parse.next(); - }); -} - -function *parseAllFiles(){ - - // Loop through local items - while(yetAnotherArrayOfSongs.length > 0) { - var file = yetAnotherArrayOfSongs.pop(); - - var resultX = yield parseFile(file); - - } - - insertEntries(); - scanLock = false; -} - - -var parse; - - - -// Insert -function insertEntries(){ - var sql2 = "insert into items (title,artist,year,album,path,format, track, disk) values "; - var sqlParser = []; - - while(arrayOfSongs.length > 0) { - var song = arrayOfSongs.pop(); - - // console.log(song); - - - var songTitle = null; - var songYear = null; - var songAlbum = null; - var artistString = null; - - if(song.artist && song.artist.length > 0){ - artistString = ''; - for (var i = 0; i < song.artist.length; i++) { - artistString += song.artist[i] + ', '; - } - artistString = artistString.slice(0, -2); - } - if(song.title && song.title.length > 0){ - songTitle = song.title; - } - if(song.year && song.year.length > 0){ - songYear = song.year; - } - if(song.album && song.album.length > 0){ - songAlbum = song.album; - } - - - sql2 += "(?, ?, ?, ?, ?, ?, ?, ?), "; - sqlParser.push(songTitle); - sqlParser.push(artistString); - sqlParser.push(songYear); - sqlParser.push(songAlbum); - sqlParser.push(song.filePath); - sqlParser.push(song.format); - sqlParser.push(song.track.no); - sqlParser.push(song.disk.no); - - } - - sql2 = sql2.slice(0, -2); - sql2 += ";"; - - console.log(sql2); - dbCopy.run(sql2, sqlParser); -} - - -// Count all files -function countFiles (dir, fileTypesArray) { - console.log('efwefwf'); - var files = fs.readdirSync( dir ); - console.log(files); - - - for (var i=0; i < files.length; i++) { - var filePath = fe.join(dir, files[i]); - var stat = fs.statSync(filePath); - console.log(filePath); - - - if(stat.isDirectory()){ - console.log('qqq'); - - countFiles(filePath , fileTypesArray); - }else{ - console.log('www'); - - var extension = getFileType(files[i]); - - if (fileTypesArray.indexOf(extension) > -1 ) { - - yetAnotherArrayOfSongs.push(filePath); - } - } - } -} - - -function runOnStart(){ - // If we are not using Beets DB, we need to prep the DB - dbCopy.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { - // console.log('TABLES CREATED'); - }); - - // Create a playlist table - dbCopy.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, user varchar, created datetime default current_timestamp);", function() { - // console.log('PLAYLIST TABLE CREATED'); - }); - - // Loop through users - - // Scan once at a time - - -} - - - -exports.setup = function(mstream, users, db){ - const rootDirCopy = rootDir; - dbCopy = db; - - runOnStart(); - - // scan and screate database - mstream.get('/db/recursive-scan', function(req,res){ - - // Check if this is already running - if(scanLock === true){ - // Return error - res.status(401).send('{"error":"Scan in progress"}'); - return; - } - - try{ - // turn on scan lock - scanLock = true; - - // Make sure directory exits - var fileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; - - countFiles(rootDirCopy, fileTypesArray); - totalFileCount = yetAnotherArrayOfSongs.length; - - dbCopy.serialize(function() { - // These two queries will run sequentially. - dbCopy.run("drop table if exists items;"); - dbCopy.run("CREATE TABLE items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { - // These queries will run in parallel and the second query will probably - // fail because the table might not exist yet. - console.log('TABLES CREATED'); - - parse = parseAllFiles(); - parse.next(); - }); - }); - - }catch(err){ - // Remove lock - scanLock = false; - - // TODO Log error - // console.log(err); - return; - } - - res.send("YA DID IT"); - - }); - - - - mstream.get('/db/status', function(req, res){ - var returnObject = {}; - - returnObject.locked = scanLock; - - - if(scanLock){ - - returnObject.totalFileCount = totalFileCount; - returnObject.filesLeft = yetAnotherArrayOfSongs.length; - - res.json(returnObject); - - }else{ - var sql = 'SELECT Count(*) FROM items'; - - dbCopy.get(sql, function(err, row){ - if(err){ - console.log(err.message); - - res.status(500).json({ error: err.message }); - return; - } - - - var fileCountDB = row['Count(*)']; // TODO: Is this correct??? - - returnObject.totalFileCount = fileCountDB; - res.json(returnObject); - - }); - } - - }); - -} diff --git a/modules/database-master.js b/modules/database-master.js deleted file mode 100644 index a1f9b7c..0000000 --- a/modules/database-master.js +++ /dev/null @@ -1,54 +0,0 @@ -exports.setup = function(mstream, users, publicDBType){ - // Go through all vars and determine plugins needed - var privateDB = false; - for (i = 0; i < program.users.length; i++) { - // Check for beets - // privateDB = sqlite3; - } - // sqlite3, mysql, sequelize, lokiJS, (? posgres) - // This will allow us to make sqlite3 an optional dependancy once lokiJS works - // Make sure all users have uniform settings for the inital run, we'll handle mixed settings later - - - - - - // Load the public DB plugin - // sqlite3 and mysql to start, add lokiJS support latet (and maybe posgres later), maybe a NO DB option even later -// The following api calls are all handled on a public level -// They can be moved into a public plugin -// pull plugin from masterDBType -const mstreamPublicDB = require('./modules/database-public-'+publicDBType+'.js'); -mstreamPublicDB.setup(mstream); - - - - - -// TODO: Load any plugins necessary for habdling indivudal user dbs - // Then construct routing between api calls and userDB management functions - - // TODO: Handle DB write functions - mstream.get('/db/status', function(req, res){ - }); - mstream.get('/db/recursive-scan', function(req,res){ - }); - - // TODO: Handle Specialized DB Functions - // mstream.get('/db/download-db', function(req, res){ - // }); - // mstream.get( '/db/hash', function(req, res){ - // }); - -} - - - -// Case 1: totalally managed by mstream, sqlite3 for everything - -// Case 2: beets DB for every user, mysql for public DB - -// Case 3: totally managed by mstream, lokiJS for everything - -// Next step: mixed user settings -// Next step: backup as local DB and import from backup diff --git a/modules/database-private-mstream.js b/modules/database-private-mstream.js deleted file mode 100644 index 49c9326..0000000 --- a/modules/database-private-mstream.js +++ /dev/null @@ -1,5 +0,0 @@ -// Function that scans user folder and returns array of all new songs to process - // Make this a generator so it can work in pieces - -// Function that can spawn new process for scannign user folders - // Queue scans so we don't start too many at a time diff --git a/modules/database-public-sqlite.js b/modules/database-public-sqlite.js deleted file mode 100644 index 5bc1f8f..0000000 --- a/modules/database-public-sqlite.js +++ /dev/null @@ -1,36 +0,0 @@ -// function that takes in a json array of songs and saves them to the sqlite db - // must contain the username and filepath for each song - -// function that gets artist info and returns json array of albums -// function that searches db and returns json array of albums and artists -// function that takes ina playlsit name and searchs db for that playlist and returns a json array of songs for that playlist -// BASICALLY, all the functions we have no but de-couple them from the Express API calls - - -exports.setup = function(mstream){ - // Attach API calls to functions - mstream.get('/getallplaylists', function (req, res){ - // Check user db settings - // pull function name out of user settings - // launch funtion - // Example functions: saveViamysql, saveViaSqlite, saveViaLoki, saveToFile - }); - mstream.get('/loadplaylist', function (req, res){ - }); - mstream.get('/deleteplaylist', function(req, res){ - }); - - - mstream.post('/db/search', function(req, res){ - }); - mstream.get('/db/artists', function (req, res) { - }); - mstream.post('/db/artists-albums', function (req, res) { - }); - mstream.get('/db/albums', function (req, res) { - }); - mstream.post('/db/album-songs', function (req, res) { - }); -//================================================================================ - -} diff --git a/modules/db-management/database-beets-manager.js b/modules/db-management/database-beets-manager.js new file mode 100644 index 0000000..77a70c7 --- /dev/null +++ b/modules/db-management/database-beets-manager.js @@ -0,0 +1,115 @@ +// // TODO: This thing has to be tested +// +// // TODO: These functions are for interacting withe the beets DB +// // Includes: rescan DB, hash files +// // Once DB has been updated, call functiosn in /db-write/database-beets-[mysql/sqlite/loki].js to pull info into publicDB +// +// const spawn = require('child_process').spawn; +// var scanLock = false; +// var yetAnotherArrayOfSongs = []; +// var totalFileCount = 0; +// +// exports.setup = function(mstream, program, rootDir, db){ +// const scanThisDir = program.beetspath; // TODO: Check that this is a real directory +// +// +// mstream.get('/db/recursive-scan-beets', function(req,res){ +// +// if(scanLock === true){ +// // Return error +// res.status(401).send('{"error":"Scan in progress"}'); +// return; +// } +// +// scanLock = true; +// var cmd = spawn('beet', [ 'import', '-A', '--group-albums' , scanThisDir]); +// +// cmd.stdout.on('data', (data) => { +// console.log(`stdout: ${data}`); +// }); +// +// cmd.stderr.on('data', (data) => { +// console.log(`stderr: ${data}`); +// scanLock = false; +// +// }); +// +// cmd.on('close', (code) => { +// console.log(`child process exited with code ${code}`); +// hashFileBeets(); +// +// // TODO: Remove all empty dirs +// }); +// }); +// +// +// function hashFileBeets(){ +// // var hashCmd = spawn('beet check -a'); +// var hashCmd = spawn('beet', [ 'check', '-a']); +// +// +// hashCmd.stdout.on('data', (data) => { +// console.log(`stdout: ${data}`); +// }); +// +// hashCmd.stderr.on('data', (data) => { +// console.log(`stderr: ${data}`); +// scanLock = false; +// +// }); +// +// hashCmd.on('close', (code) => { +// console.log(`child process exited with code ${code}`); +// scanLock = false; +// +// }); +// } +// +// // TODO: Function that will remove all empty folders +// function removeEmptyFolders(){ +// var hashCmd = spawn('beet', [ 'check', '-a']); +// // 'find ~ -type d -empty -delete' +// } +// +// +// +// mstream.get('/db/status-beets', function(req, res){ +// var returnObject = {}; +// +// returnObject.locked = scanLock; +// +// +// if(scanLock){ +// +// // Currently we don't support filecount stats when using beets DB +// // Dummy data +// returnObject.totalFileCount = 0; +// returnObject.filesLeft = 0; +// +// +// res.json(returnObject); +// +// }else{ +// var sql = 'SELECT Count(*) FROM items'; +// +// db.get(sql, function(err, row){ +// if(err){ +// console.log(err.message); +// +// res.status(500).json({ error: err.message }); +// return; +// } +// +// +// var fileCountDB = row['Count(*)']; // TODO: Is this correct??? +// +// returnObject.totalFileCount = fileCountDB; +// res.json(returnObject); +// +// }); +// } +// +// }); +// +// +// } diff --git a/modules/db-management/database-default-manager.js b/modules/db-management/database-default-manager.js new file mode 100644 index 0000000..5932542 --- /dev/null +++ b/modules/db-management/database-default-manager.js @@ -0,0 +1,574 @@ +const metadata = require('musicmetadata'); +const fs = require('graceful-fs'); // File System +const fe = require('path'); +const crypto = require('crypto'); + +// var dbCopy; +// +// +// var arrayOfSongs = []; +// var scanLock = false; +// var yetAnotherArrayOfSongs = []; +// var totalFileCount = 0; +// + + + +//TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: + +// Break this into two pieces + +// This piece will contain all the functions that scan for files and parse meta data and hashFileBeets + // These functions will return JSON arrays of song data + +// The next piece will contain the all the functions that store data into the db + // These functions will take in JSON arrays of song data and then save that dat to the DB + // next piece name: /modules/db-write/database-default-[sqlite/mysql/loki].js + +//TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: + + + + + +//TODO: Spawn new thread for processing files +// Handle users when processing files +// Hash files when processing +// Handle album art when processing files +// Use created/modified dates to handle updating DB + + +// function getFileType(filename){ +// return filename.split(".").pop(); +// } +// +// function parseFile(thisSong){ +// var readableStream = fs.createReadStream(thisSong); +// var parser = metadata(readableStream, function (err, songInfo) { +// if(err){ +// // TODO: Do something +// } +// +// +// // TODO: Hash the file here and add the hash to the DB +// +// console.log(songInfo); +// +// +// // Close the stream +// readableStream.close(); +// +// +// songInfo.filePath = thisSong; +// songInfo.format = getFileType(thisSong); +// +// arrayOfSongs.push(songInfo); +// +// +// // if there are more than 100 entries, or if it's the last song +// //TODO +// if(arrayOfSongs.length > 99){ +// insertEntries(); +// } +// +// // For the generator +// parse.next(); +// }); +// } +// +// function *parseAllFiles(){ +// +// // Loop through local items +// while(yetAnotherArrayOfSongs.length > 0) { +// var file = yetAnotherArrayOfSongs.pop(); +// +// var resultX = yield parseFile(file); +// +// } +// +// // TODO +// insertEntries(); +// scanLock = false; +// } +// +// +// var parse; + + + +// Insert +// TODO: Move this to db-write/database-default-X.js +// function insertEntries(){ +// var sql2 = "insert into items (title,artist,year,album,path,format, track, disk) values "; +// var sqlParser = []; +// +// while(arrayOfSongs.length > 0) { +// var song = arrayOfSongs.pop(); +// +// // console.log(song); +// +// +// var songTitle = null; +// var songYear = null; +// var songAlbum = null; +// var artistString = null; +// +// if(song.artist && song.artist.length > 0){ +// artistString = ''; +// for (var i = 0; i < song.artist.length; i++) { +// artistString += song.artist[i] + ', '; +// } +// artistString = artistString.slice(0, -2); +// } +// if(song.title && song.title.length > 0){ +// songTitle = song.title; +// } +// if(song.year && song.year.length > 0){ +// songYear = song.year; +// } +// if(song.album && song.album.length > 0){ +// songAlbum = song.album; +// } +// +// +// sql2 += "(?, ?, ?, ?, ?, ?, ?, ?), "; +// sqlParser.push(songTitle); +// sqlParser.push(artistString); +// sqlParser.push(songYear); +// sqlParser.push(songAlbum); +// sqlParser.push(song.filePath); +// sqlParser.push(song.format); +// sqlParser.push(song.track.no); +// sqlParser.push(song.disk.no); +// +// } +// +// sql2 = sql2.slice(0, -2); +// sql2 += ";"; +// +// console.log(sql2); +// dbCopy.run(sql2, sqlParser); +// } + +// +// // Count all files +// function countFiles (dir, fileTypesArray) { +// console.log('efwefwf'); +// var files = fs.readdirSync( dir ); +// console.log(files); +// +// +// for (var i=0; i < files.length; i++) { +// var filePath = fe.join(dir, files[i]); +// var stat = fs.statSync(filePath); +// console.log(filePath); +// +// +// if(stat.isDirectory()){ +// console.log('qqq'); +// +// countFiles(filePath , fileTypesArray); +// }else{ +// console.log('www'); +// +// var extension = getFileType(files[i]); +// +// if (fileTypesArray.indexOf(extension) > -1 ) { +// +// yetAnotherArrayOfSongs.push(filePath); +// } +// } +// } +// } + +// +// function runOnStart(){ +// +// // Loop through users +// // Scan one at a time +// +// // Check DB for the last addition timed +// +// // Loop through users files +// +// // Check for modification time +// // if modification time is newer than the latest time, go to next step +// // if file exists inthe db already, add it to modificationCheckArray +// // if it doesn't exist, add it to newSongArray +// +// } + + + +exports.setup = function(mstream, users, db){ + const rootDirCopy = rootDir; + dbCopy = db; + + runOnStart(); + + // scan and screate database + mstream.get('/db/recursive-scan-mstream', function(req,res){ + + // Check if this is already running + if(scanLock === true){ + // Return error + res.status(401).send('{"error":"Scan in progress"}'); + return; + } + + try{ + // turn on scan lock + scanLock = true; + + // Make sure directory exits + var fileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; + + countFiles(rootDirCopy, fileTypesArray); + totalFileCount = yetAnotherArrayOfSongs.length; + + // TODO: Move this + // dbCopy.serialize(function() { + // // These two queries will run sequentially. + // dbCopy.run("drop table if exists items;"); + // dbCopy.run("CREATE TABLE items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { + // // These queries will run in parallel and the second query will probably + // // fail because the table might not exist yet. + // console.log('TABLES CREATED'); + // + // parse = parseAllFiles(); + // parse.next(); + // }); + // }); + + }catch(err){ + // Remove lock + scanLock = false; + + // TODO Log error + // console.log(err); + return; + } + + res.send("YA DID IT"); + + }); + + + + mstream.get('/db/status-mstream', function(req, res){ + var returnObject = {}; + + returnObject.locked = scanLock; + + + if(scanLock){ + + returnObject.totalFileCount = totalFileCount; + returnObject.filesLeft = yetAnotherArrayOfSongs.length; + + res.json(returnObject); + + }else{ + var sql = 'SELECT Count(*) FROM items'; + + dbCopy.get(sql, function(err, row){ + if(err){ + console.log(err.message); + + res.status(500).json({ error: err.message }); + return; + } + + + var fileCountDB = row['Count(*)']; // TODO: Is this correct??? + + returnObject.totalFileCount = fileCountDB; + res.json(returnObject); + + }); + } + + }); + +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +// 2.0 + // Get all files from DB + // Hash file. If file and hash are found in array, then skip + // Seperate into new files and files that need to be updated + // Send these arrays to functions in database-default-X.js + + +var arrayOfSongs; // Holds songs for DB to process // TODO: Move out of global scope +var arrayOfScannedFiles = []; // Holds files for from recursive scan + +var parseFilesGenerator; // This Generator is used in two places. Should it be seperated? +var scanDirLock = false; + +//TODO: Pull in correct module +const dbRead = require('./modules/db-write/database-default-sqlite.js') + +function rescanAllDirectoriesWrapper(){ + if(scanDirLock === true){ + // TODO: If scanlock == true, aleart user to try again once scanning is done + // TODO: Enable Button + return; + } + + scanDirLock = true; + // TODO: Disable Button + + parseFilesGenerator = rescanAllDirectories(dir); + parseFilesGenerator.next(); +} + + + +function *rescanAllDirectories(directoryToScan){ + + // Scan the directory for new, modified, and deleted files + var filesToProcess = yield rescanDirectory(directoryToScan); + + // Process all new files + while(filesToProcess.newFiles.length > 0) { + // TODO: Break into chuncks and send to dbRead + yield parseFile(filesToProcess.newFiles.pop()); + } + + // process all updated files + while(filesToProcess.updatedFiles.length > 0) { + // TODO: Break into chuncks and send to dbRead + yield hashOneUpdatedSong(filesToProcess.updatedFiles.pop()); + } + + // Re-enable scanning + scanDirLock = false; +} + + + +function rescanDirectory(dir){ + + // Get all files from DB + // TODO: Move This + dbRead.getUserFiles(user, function(rows){ + + // Scan through files + var fileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; + recursiveScan(dir, fileTypesArray); + + var latestFileList = arrayOfScannedFiles; + arrayOfScannedFiles = []; + + + + var dbFileList = []; + var dbFileListTimestamp = []; + + for(var s of rows){ + console.log(s); + dbFileList.push(s.path); + + dbFileListTimestamp.push(s.path + '::' + s.file_modified_date); + } + + console.log(dbFileList); + console.log(dbFileListTimestamp); + + + + var arrayOfUpdatedSongsToProcess = []; + var arrayOfSongsToProcess = []; + var deletedFiles = []; // TODO: Global variable ??? + var checkForModifications = []; + + + + var latestFileListSet = new Set(latestFileList); + var dbFileListSet = new Set(dbFileList); + var dbFileListTimestampSet = new Set(dbFileListTimestamp); + + + // Get deleted files + dbFileList.filter(function(x) { + if( latestFileListSet.has(x) ){ + + }else{ + // It's deleted ( i think) + deletedFiles.push(x); + } + }); + + console.log('DELETED FILES'); + console.log(deletedFiles); + + + // Get new files + latestFileList.filter(function(x) { + + console.log(x); + + if(dbFileListSet.has(x)){ + checkForModifications.push(x + "::" + fs.statSync(x).mtime.getTime() ); + console.log('yes'); + + }else{ + // New files + arrayOfSongsToProcess.push(x); + console.log('no'); + + } + }); + + + console.log('NEW FILES'); + console.log(arrayOfSongsToProcess); + + // loop through checkForModifications + // Append timestamp to all path strings + // Compare dbFileList clone with path strings appended + checkForModifications.filter(function(x) { + if(dbFileListTimestampSet.has(x)){ + + }else{ + // File x has been updated + var filePath = x.split("::")[0]; + arrayOfUpdatedSongsToProcess.push(filePath); + } + }); + + console.log('POTENTIALLY UPDATED SONGS'); + console.log(arrayOfUpdatedSongsToProcess); + + + + // TODO: Handle deleted files + // We need to prompt users to see if they want to delete files on the server side + // We can store a default behaviour + + returnArray = { + "newFiles":arrayOfSongsToProcess, + "updatedFiles":arrayOfUpdatedSongsToProcess, + "deletedFiles":deletedFiles + }; + parseFilesGenerator.next(returnArray); + }); + +} + + + +function parseFile(thisSong){ + var filestat = fs.statSync(thisSong); + if(!filestat.isFile()){ + // TODO: Something is fucky, log it + parseFilesGenerator.next(); + return; + } + + // Stores all data that needs to be added to DB + var songInfo; + + // TODO: Hash the file here and add the hash to the DB + var hash = crypto.createHash('sha256'); + hash.setEncoding('hex'); + + var readableStream = fs.createReadStream(thisSong); + var parser = metadata(readableStream, function (err, songInfo) { + if(err){ + // TODO: Do something + } + songInfo = thisSong; + songInfo.filesize = filestat.size; + songInfo.created = filestat.birthtime.getTime(); + songInfo.modified = ilestat.mtime.getTime(); + songInfo.filePath = thisSong; + songInfo.format = getFileType(thisSong); + + + + readableStream.on('end', function () { + hash.end(); + readableStream.close(); + + songInfo.hash = String(hash.read()); + arrayOfSongs.push(songInfo); + + // if there are more than 100 entries, or if it's the last song + if(arrayOfSongs.length > 99){ + //TODO: Need to move this function + insertEntries(); + } + + // For the generator + parseFilesGenerator.next(); + }); + + + }); + readableStream.pipe(hash); +} + +function getFileType(filename){ + return filename.split(".").pop(); +} + +function recursiveScan(dir, fileTypesArray){ + var files = fs.readdirSync( dir ); + + + // loop through files + for (var i=0; i < files.length; i++) { + // var filePath = dir + files[i]; + var filePath = fe.join(dir, files[i]); + // console.log(filePath); + var stat = fs.statSync(filePath); + + + if(stat.isDirectory()){ + recursiveScan(filePath, fileTypesArray); + }else{ + var extension = getFileType(files[i]); + + // Make sure this is in our list of allowed files + if (fileTypesArray.indexOf(extension) > -1 ) { + arrayOfScannedFiles.push(filePath); + } + } + } +} diff --git a/modules/db-management/database-master.js b/modules/db-management/database-master.js new file mode 100644 index 0000000..3adad60 --- /dev/null +++ b/modules/db-management/database-master.js @@ -0,0 +1,89 @@ +const spawn = require('child_process').spawn; + +exports.setup = function(mstream, users, publicDBType){ + + // sqlite3, mysql, sequelize, lokiJS, (? posgres) + // This will allow us to make sqlite3 an optional dependancy once lokiJS works + // Make sure all users have uniform settings for the inital run, we'll handle mixed settings later + + + + + + // Load the public DB plugin + // sqlite3 and mysql to start, add lokiJS support latet (and maybe posgres later), maybe a NO DB option even later + // The following api calls are all handled on a public level + // They can be moved into a public plugin + // pull plugin from masterDBType + const mstreamReadPublicDB = require('./modules/db-read/database-public-'+publicDBType+'.js'); + mstreamReadPublicDB.setup(mstream); + + + + mstream.get('/db/recursive-scan', function(req,res){ + // Get user's db setup + + // spawn a child_process to scan + // spawnBeets(user, userCommand); // For beets we need pull the exact command to launch from the user config + // spawnDefault(user); + }); + + function spawnDefault(user){ + + } + + + // TODO: Handle user status + mstream.get('/db/status-mstream', function(req, res){ + }); + + +// ======================================================================================== +// // Either copy from sqliteDB or use built-in functions +// // Go through all vars and determine plugins needed +// var useMstream = false; +// var useBeets = false; +// +// for (i = 0; i < program.users.length; i++) { +// // Check for beets +// // var useMstream = true; +// // var useBeets = true; +// } +// if(useMstream){ +// const mstreamWritePublicDB = require('./modules/database-default-'+publicDBType+'.js'); // FIXME; Rename +// mstreamWritePublicDB.setup(mstream, users, db); +// } +// if(useBeets){ +// const beetsWritePublicDB = require('./modules/database-beets-'+publicDBType+'.js'); // FIXME; Rename +// } +// ======================================================================================== + + + + + + + + +// TODO: Load any plugins necessary for habdling indivudal user dbs + // Then construct routing between api calls and userDB management functions + + + // TODO: Handle Specialized DB Functions + // mstream.get('/db/download-db', function(req, res){ + // }); + // mstream.get( '/db/hash', function(req, res){ + // }); + +} + + + +// Case 1: totalally managed by mstream, sqlite3 for everything + +// Case 2: beets DB for every user, mysql for public DB + +// Case 3: totally managed by mstream, lokiJS for everything + +// Next step: mixed user settings +// Next step: backup as local DB and import from backup diff --git a/modules/db-read/database-public-sqlite.js b/modules/db-read/database-public-sqlite.js new file mode 100644 index 0000000..889a17e --- /dev/null +++ b/modules/db-read/database-public-sqlite.js @@ -0,0 +1,263 @@ +// function that takes in a json array of songs and saves them to the sqlite db + // must contain the username and filepath for each song + +// function that gets artist info and returns json array of albums +// function that searches db and returns json array of albums and artists +// function that takes ina playlsit name and searchs db for that playlist and returns a json array of songs for that playlist +// BASICALLY, all the functions we have no but de-couple them from the Express API calls + +// TODO: Setup SQLite + + +function getFileType(filename){ + return filename.split(".").pop(); +} + + +exports.setup = function(mstream){ + db.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER, user VARCHAR);", function() { + // console.log('TABLES CREATED'); + }); + // Create a playlist table + db.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, user VARCHAR, created datetime default current_timestamp);", function() { + // console.log('PLAYLIST TABLE CREATED'); + }); + + + + // TODO: This needs to be tested to see if it works on extra large playlists (think thousands of entries) + // TODO: Ban saving playlists that are > 10,000 items long + mstream.post('/saveplaylist', function (req, res){ + var title = req.body.title; + var songs = req.body.stuff; + + // Check if this playlist already exists + db.all("SELECT id FROM mstream_playlists WHERE playlist_name = ? AND user = ?;", [title, req.user.username], function(err, rows) { + + db.serialize(function() { + + // We need to delete anys existing entries + if(rows && rows.length > 0){ + db.run("DELETE FROM mstream_playlists WHERE playlist_name = ? AND user = ?;", [title, req.user.username]); + } + + // Now we add the new entries + var sql2 = "insert into mstream_playlists (playlist_name, filepath, user) values "; + var sqlParser = []; + + while(songs.length > 0) { + var song = songs.shift(); + + sql2 += "(?, ?, ?), "; + sqlParser.push(title); + sqlParser.push( fe.join(req.user.musicDir, song) ); // TODO: User music dir + sqlParser.push( req.user.username ); // TODO: User music dir + + } + + sql2 = sql2.slice(0, -2); + sql2 += ";"; + + db.run(sql2, sqlParser, function(){ + res.send('DONE'); + }); + + }); + }); + }); + + + // Attach API calls to functions + mstream.get('/getallplaylists', function (req, res){ + // TODO: In V2 we need to change this to ignore hidden playlists + // TODO: db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE hide=0;", function(err, rows){ + db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE user = ?", [req.user.username], function(err, rows){ + var playlists = []; + + // loop through files + for (var i = 0; i < rows.length; i++) { + if(rows[i].playlist_name){ + playlists.push({name: rows[i].playlist_name}); + } + } + + res.send(JSON.stringify(playlists)); + }); + }); + mstream.get('/loadplaylist', function (req, res){ + var playlist = req.query.playlistname; + + db.all("SELECT * FROM mstream_playlists WHERE playlist_name = ? ORDER BY id COLLATE NOCASE ASC", [playlist, req.user.username], function(err, rows){ + var returnThis = []; + + for (var i = 0; i < rows.length; i++) { + + // var tempName = rows[i].filepath.split('/').slice(-1)[0]; + var tempName = fe.basename(rows[i].filepath); + var extension = getFileType(rows[i].filepath); + var filepath = slash(fe.relative(req.user.musicDir, rows[i].filepath)); // TODO + + returnThis.push({name: tempName, file: filepath, filetype: extension }); + } + + res.send(JSON.stringify(returnThis)); + }); + }); + mstream.get('/deleteplaylist', function(req, res){ + var playlistname = req.query.playlistname; + + // Handle a soft delete + if(req.query.hide && parseInt(req.query.hide) === 1 ){ + db.run("UPDATE mstream_playlists SET hide = 1 WHERE playlist_name = ? AND user = ?;", [playlistname, req.user.username], function(){ + res.send('DONE'); + + }); + }else{ // Permentaly delete + + // Delete playlist from DB + db.run("DELETE FROM mstream_playlists WHERE playlist_name = ? AND user = ?;", [playlistname, req.user.username], function(){ + res.send('DONE'); + + }); + } + }); + + + mstream.post('/db/search', function(req, res){ + var searchTerm = "%" + req.body.search + "%" ; + + var returnThis = {"albums":[], "artists":[]}; + + // TODO: Combine SQL calls into one + db.serialize(function() { + + var sqlAlbum = "SELECT DISTINCT album FROM items WHERE items.album LIKE ? AND user = ? ORDER BY album COLLATE NOCASE ASC;"; + db.all(sqlAlbum, [searchTerm, req.user.username], function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + for (var i = 0; i < rows.length; i++) { + if(rows[i].album){ + returnThis.albums.push(rows[i].album); + } + } + }); + + + var sqlArtist = "SELECT DISTINCT artist FROM items WHERE items.artist LIKE ? ORDER BY artist COLLATE NOCASE ASC;"; + db.all(sqlArtist, [searchTerm, req.user.username], function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + for (var i = 0; i < rows.length; i++) { + if(rows[i].artist){ + returnThis.artists.push(rows[i].artist); + } + } + + res.send(JSON.stringify(returnThis)); + }); + }); + }); + + mstream.get('/db/artists', function (req, res) { + var artists = {"artists":[]}; + + var sql = "SELECT DISTINCT artist FROM items WHERE user = ? ORDER BY artist COLLATE NOCASE ASC;"; + db.all(sql, [req.user.username], function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + var returnArray = []; + for (var i = 0; i < rows.length; i++) { + if(rows[i].artist){ + // rows.splice(i, 1); + artists.artists.push(rows[i].artist); + } + } + + res.send(JSON.stringify(artists)); + }); + }); + + mstream.post('/db/artists-albums', function (req, res) { + var albums = {"albums":[]}; + + // TODO: Make a list of all songs without null albums and add them to the response + var sql = "SELECT DISTINCT album FROM items WHERE artist = ? AND user = ? ORDER BY album COLLATE NOCASE ASC;"; + var searchTerms = []; + searchTerms.push(req.body.artist); + searchTerms.push(req.user.username); + + db.all(sql, searchTerms, function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + var returnArray = []; + for (var i = 0; i < rows.length; i++) { + if(rows[i].album){ + albums.albums.push(rows[i].album); + } + } + + res.send(JSON.stringify(albums)); + }); + }); + + mstream.get('/db/albums', function (req, res) { + var albums = {"albums":[]}; + + var sql = "SELECT DISTINCT album FROM items ORDER BY album COLLATE NOCASE ASC WHERE user = ?;"; + db.all(sql, [req.user.username], function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + var returnArray = []; + for (var i = 0; i < rows.length; i++) { + if(rows[i].album){ + albums.albums.push(rows[i].album); + } + } + + res.send(JSON.stringify(albums)); + }); + }); + + mstream.post('/db/album-songs', function (req, res) { + var sql = "SELECT title, artist, album, format, year, cast(path as TEXT), track FROM items WHERE album = ? AND user = ? ORDER BY track ASC;"; + + var searchTerms = []; + searchTerms.push(req.body.album); + searchTerms.push(req.user.username); + + db.all(sql, searchTerms, function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + // Format data for API + // rows = setLocalFileLocation(rows); + for(var i in rows ){ + var path = String(rows[i]['cast(path as TEXT)']); + + rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase + rows[i].file_location = slash(fe.relative(req.user.musicDir, path)); // Get the local file location + rows[i].filename = fe.basename( path ); // Ge the filname + } + + res.send(JSON.stringify(rows)); + }); + }); + +} diff --git a/modules/db-write/database-beets-mysql.js b/modules/db-write/database-beets-mysql.js new file mode 100644 index 0000000..ce55153 --- /dev/null +++ b/modules/db-write/database-beets-mysql.js @@ -0,0 +1 @@ +// TODO: Function that copies a BeetsDB(private) into the MySQL master(public) DB diff --git a/modules/db-write/database-beets-sqlite.js b/modules/db-write/database-beets-sqlite.js new file mode 100644 index 0000000..92e8106 --- /dev/null +++ b/modules/db-write/database-beets-sqlite.js @@ -0,0 +1 @@ +// TODO: Function that copies a BeetsDB(private) into the SQLite master(public) DB diff --git a/modules/db-write/database-default-mysql.js b/modules/db-write/database-default-mysql.js new file mode 100644 index 0000000..b25a5c9 --- /dev/null +++ b/modules/db-write/database-default-mysql.js @@ -0,0 +1,2 @@ +// functions that store data into the MySQL DB + // These functions will take in JSON arrays of song data and then save that dat to the DB diff --git a/modules/db-write/database-default-sqlite.js b/modules/db-write/database-default-sqlite.js new file mode 100644 index 0000000..375c0e9 --- /dev/null +++ b/modules/db-write/database-default-sqlite.js @@ -0,0 +1,82 @@ +// functions that store data into the SQLite DB + // These functions will take in JSON arrays of song data and then save that dat to the DB + + +exports.getUserFiles = function(user, callback){ + db.all("SELECT path, file_modified_date FROM files WHERE user=? ;" [thisUser], function(err, rows){ + // Format results + var returnThis; + + // callback function + callback(returnThis); + }); +} + + + +function insertEntries(){ + var sql2 = "insert into items (title,artist,year,album,path,format, track, disk) values "; + var sqlParser = []; + + while(arrayOfSongs.length > 0) { + var song = arrayOfSongs.pop(); + + // console.log(song); + + + var songTitle = null; + var songYear = null; + var songAlbum = null; + var artistString = null; + + if(song.artist && song.artist.length > 0){ + artistString = ''; + for (var i = 0; i < song.artist.length; i++) { + artistString += song.artist[i] + ', '; + } + artistString = artistString.slice(0, -2); + } + if(song.title && song.title.length > 0){ + songTitle = song.title; + } + if(song.year && song.year.length > 0){ + songYear = song.year; + } + if(song.album && song.album.length > 0){ + songAlbum = song.album; + } + + + sql2 += "(?, ?, ?, ?, ?, ?, ?, ?), "; + sqlParser.push(songTitle); + sqlParser.push(artistString); + sqlParser.push(songYear); + sqlParser.push(songAlbum); + sqlParser.push(song.filePath); + sqlParser.push(song.format); + sqlParser.push(song.track.no); + sqlParser.push(song.disk.no); + + } + + sql2 = sql2.slice(0, -2); + sql2 += ";"; + + console.log(sql2); + dbCopy.run(sql2, sqlParser); +} + +function prep(){ + dbCopy.serialize(function() { + // These two queries will run sequentially. + dbCopy.run("drop table if exists items;"); + dbCopy.run("CREATE TABLE items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { + // These queries will run in parallel and the second query will probably + // fail because the table might not exist yet. + console.log('TABLES CREATED'); + + parse = parseAllFiles(); + parse.next(); + }); + }); +} diff --git a/mstream.js b/mstream.js index 38f97a2..e761140 100755 --- a/mstream.js +++ b/mstream.js @@ -289,7 +289,7 @@ if(program.users){ } var user = Users[username]; - user['id'] = username; + user['username'] = username; // Make a token for the user var token = jwt.sign(user, secret); @@ -614,7 +614,7 @@ mstream.post('/download', function (req, res){ // New Way var publicDBType = 'sqlite3'; // Can be sqlite3/mysql/LokiJS -const mstreamDB = require('./modules/database-master.js'); +const mstreamDB = require('./modules/db-management/database-master.js'); mstreamDB.setup(mstream, program.users, publicDBType); @@ -664,132 +664,129 @@ mstream.post('/db/search', function(req, res){ -mstream.get('/db/artists', function (req, res) { - var sql = "SELECT DISTINCT artist FROM items ORDER BY artist COLLATE NOCASE ASC;"; - - var artists = {"artists":[]}; - - db.all(sql, function(err, rows) { - if(err){ - res.status(500).json({ error: 'DB Error' }); - return; - } - - var returnArray = []; - for (var i = 0; i < rows.length; i++) { - if(rows[i].artist){ - // rows.splice(i, 1); - artists.artists.push(rows[i].artist); - } - } - - res.send(JSON.stringify(artists)); - }); -}); - - - -mstream.post('/db/artists-albums', function (req, res) { - var sql = "SELECT DISTINCT album FROM items WHERE artist = ? ORDER BY album COLLATE NOCASE ASC;"; - - var searchTerm = req.body.artist ; - - var albums = {"albums":[]}; - - // TODO: Make a list of all songs without null albums and add them to the response - - - db.all(sql, searchTerm, function(err, rows) { - if(err){ - res.status(500).json({ error: 'DB Error' }); - return; - } - - - var returnArray = []; - for (var i = 0; i < rows.length; i++) { - if(rows[i].album){ - // rows.splice(i, 1); - albums.albums.push(rows[i].album); - } - } - - res.send(JSON.stringify(albums)); - }); -}); - - - -mstream.get('/db/albums', function (req, res) { - var sql = "SELECT DISTINCT album FROM items ORDER BY album COLLATE NOCASE ASC;"; - - var albums = {"albums":[]}; - - - db.all(sql, function(err, rows) { - if(err){ - res.status(500).json({ error: 'DB Error' }); - return; - } - - - var returnArray = []; - for (var i = 0; i < rows.length; i++) { - if(rows[i].album){ - albums.albums.push(rows[i].album); - - } - } - - console.log(JSON.stringify(albums)); - res.send(JSON.stringify(albums)); - }); -}); - - - -mstream.post('/db/album-songs', function (req, res) { - var sql = "SELECT title, artist, album, format, year, cast(path as TEXT), track FROM items WHERE album = ? ORDER BY track ASC;"; - var searchTerm = req.body.album ; - - - - db.all(sql, searchTerm, function(err, rows) { - if(err){ - res.status(500).json({ error: 'DB Error' }); - return; - } - - // Format data for API - // rows = setLocalFileLocation(rows); - for(var i in rows ){ - var path = String(rows[i]['cast(path as TEXT)']); - - rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase - rows[i].file_location = slash(fe.relative(req.user.musicDir, path)); // Get the local file location - rows[i].filename = fe.basename( path ); // Ge the filname - } - - - res.send(JSON.stringify(rows)); - }); -}); - - - -// // TODO -// function setLocalFileLocation(rows){ +// mstream.get('/db/artists', function (req, res) { +// var sql = "SELECT DISTINCT artist FROM items ORDER BY artist COLLATE NOCASE ASC;"; // -// for(var i in rows ){ -// var path = String(rows[i]['cast(path as TEXT)']); +// var artists = {"artists":[]}; // -// rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase -// rows[i].file_location = slash(fe.relative(rootDir, path)); // Get the local file location -// rows[i].filename = fe.basename( path ); // Ge the filname -// } +// db.all(sql, function(err, rows) { +// if(err){ +// res.status(500).json({ error: 'DB Error' }); +// return; +// } // -// return rows; -// } +// var returnArray = []; +// for (var i = 0; i < rows.length; i++) { +// if(rows[i].artist){ +// // rows.splice(i, 1); +// artists.artists.push(rows[i].artist); +// } +// } +// +// res.send(JSON.stringify(artists)); +// }); +// }); + + + +// mstream.post('/db/artists-albums', function (req, res) { +// var sql = "SELECT DISTINCT album FROM items WHERE artist = ? ORDER BY album COLLATE NOCASE ASC;"; +// +// var searchTerm = req.body.artist ; +// +// var albums = {"albums":[]}; +// +// // TODO: Make a list of all songs without null albums and add them to the response +// +// +// db.all(sql, searchTerm, function(err, rows) { +// if(err){ +// res.status(500).json({ error: 'DB Error' }); +// return; +// } +// +// +// var returnArray = []; +// for (var i = 0; i < rows.length; i++) { +// if(rows[i].album){ +// // rows.splice(i, 1); +// albums.albums.push(rows[i].album); +// } +// } +// +// res.send(JSON.stringify(albums)); +// }); +// }); + + + +// mstream.get('/db/albums', function (req, res) { +// var sql = "SELECT DISTINCT album FROM items ORDER BY album COLLATE NOCASE ASC;"; +// +// var albums = {"albums":[]}; +// +// +// db.all(sql, function(err, rows) { +// if(err){ +// res.status(500).json({ error: 'DB Error' }); +// return; +// } +// +// +// var returnArray = []; +// for (var i = 0; i < rows.length; i++) { +// if(rows[i].album){ +// albums.albums.push(rows[i].album); +// +// } +// } +// +// console.log(JSON.stringify(albums)); +// res.send(JSON.stringify(albums)); +// }); +// }); + + + +// mstream.post('/db/album-songs', function (req, res) { +// var sql = "SELECT title, artist, album, format, year, cast(path as TEXT), track FROM items WHERE album = ? ORDER BY track ASC;"; +// var searchTerm = req.body.album ; +// +// +// +// db.all(sql, searchTerm, function(err, rows) { +// if(err){ +// res.status(500).json({ error: 'DB Error' }); +// return; +// } +// +// // Format data for API +// // rows = setLocalFileLocation(rows); +// for(var i in rows ){ +// var path = String(rows[i]['cast(path as TEXT)']); +// +// rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase +// rows[i].file_location = slash(fe.relative(req.user.musicDir, path)); // Get the local file location +// rows[i].filename = fe.basename( path ); // Ge the filname +// } +// +// +// res.send(JSON.stringify(rows)); +// }); +// }); +// // // TODO +// // function setLocalFileLocation(rows){ +// // +// // for(var i in rows ){ +// // var path = String(rows[i]['cast(path as TEXT)']); +// // +// // rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase +// // rows[i].file_location = slash(fe.relative(rootDir, path)); // Get the local file location +// // rows[i].filename = fe.basename( path ); // Ge the filname +// // } +// // +// // return rows; +// // } From 11da492a8fda6c715abda051790f6a4b30f335ea Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Sun, 27 Nov 2016 12:54:15 -0500 Subject: [PATCH 03/11] Stashing some more changes --- .../db-management/database-default-manager.js | 343 ++---------------- modules/db-management/database-master.js | 31 +- mstream.js | 284 +++++++-------- 3 files changed, 194 insertions(+), 464 deletions(-) diff --git a/modules/db-management/database-default-manager.js b/modules/db-management/database-default-manager.js index 5932542..92e8a9e 100644 --- a/modules/db-management/database-default-manager.js +++ b/modules/db-management/database-default-manager.js @@ -1,330 +1,33 @@ +#!/usr/bin/env node + +// This is designed to run as it's own process +// It takes in a json array +// { +// "username":"lol", +// "userDir":"/path/to/dir", +// "dbType":"sqlite", +// "dbSettings":{} +// } const metadata = require('musicmetadata'); -const fs = require('graceful-fs'); // File System +const fs = require('fs'); // File System const fe = require('path'); const crypto = require('crypto'); - -// var dbCopy; -// -// -// var arrayOfSongs = []; -// var scanLock = false; -// var yetAnotherArrayOfSongs = []; -// var totalFileCount = 0; -// +const fe = require('path'); - -//TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: - -// Break this into two pieces - -// This piece will contain all the functions that scan for files and parse meta data and hashFileBeets - // These functions will return JSON arrays of song data - -// The next piece will contain the all the functions that store data into the db - // These functions will take in JSON arrays of song data and then save that dat to the DB - // next piece name: /modules/db-write/database-default-[sqlite/mysql/loki].js - -//TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: - - - - - -//TODO: Spawn new thread for processing files -// Handle users when processing files -// Hash files when processing -// Handle album art when processing files -// Use created/modified dates to handle updating DB - - -// function getFileType(filename){ -// return filename.split(".").pop(); -// } -// -// function parseFile(thisSong){ -// var readableStream = fs.createReadStream(thisSong); -// var parser = metadata(readableStream, function (err, songInfo) { -// if(err){ -// // TODO: Do something -// } -// -// -// // TODO: Hash the file here and add the hash to the DB -// -// console.log(songInfo); -// -// -// // Close the stream -// readableStream.close(); -// -// -// songInfo.filePath = thisSong; -// songInfo.format = getFileType(thisSong); -// -// arrayOfSongs.push(songInfo); -// -// -// // if there are more than 100 entries, or if it's the last song -// //TODO -// if(arrayOfSongs.length > 99){ -// insertEntries(); -// } -// -// // For the generator -// parse.next(); -// }); -// } -// -// function *parseAllFiles(){ -// -// // Loop through local items -// while(yetAnotherArrayOfSongs.length > 0) { -// var file = yetAnotherArrayOfSongs.pop(); -// -// var resultX = yield parseFile(file); -// -// } -// -// // TODO -// insertEntries(); -// scanLock = false; -// } -// -// -// var parse; - - - -// Insert -// TODO: Move this to db-write/database-default-X.js -// function insertEntries(){ -// var sql2 = "insert into items (title,artist,year,album,path,format, track, disk) values "; -// var sqlParser = []; -// -// while(arrayOfSongs.length > 0) { -// var song = arrayOfSongs.pop(); -// -// // console.log(song); -// -// -// var songTitle = null; -// var songYear = null; -// var songAlbum = null; -// var artistString = null; -// -// if(song.artist && song.artist.length > 0){ -// artistString = ''; -// for (var i = 0; i < song.artist.length; i++) { -// artistString += song.artist[i] + ', '; -// } -// artistString = artistString.slice(0, -2); -// } -// if(song.title && song.title.length > 0){ -// songTitle = song.title; -// } -// if(song.year && song.year.length > 0){ -// songYear = song.year; -// } -// if(song.album && song.album.length > 0){ -// songAlbum = song.album; -// } -// -// -// sql2 += "(?, ?, ?, ?, ?, ?, ?, ?), "; -// sqlParser.push(songTitle); -// sqlParser.push(artistString); -// sqlParser.push(songYear); -// sqlParser.push(songAlbum); -// sqlParser.push(song.filePath); -// sqlParser.push(song.format); -// sqlParser.push(song.track.no); -// sqlParser.push(song.disk.no); -// -// } -// -// sql2 = sql2.slice(0, -2); -// sql2 += ";"; -// -// console.log(sql2); -// dbCopy.run(sql2, sqlParser); -// } - -// -// // Count all files -// function countFiles (dir, fileTypesArray) { -// console.log('efwefwf'); -// var files = fs.readdirSync( dir ); -// console.log(files); -// -// -// for (var i=0; i < files.length; i++) { -// var filePath = fe.join(dir, files[i]); -// var stat = fs.statSync(filePath); -// console.log(filePath); -// -// -// if(stat.isDirectory()){ -// console.log('qqq'); -// -// countFiles(filePath , fileTypesArray); -// }else{ -// console.log('www'); -// -// var extension = getFileType(files[i]); -// -// if (fileTypesArray.indexOf(extension) > -1 ) { -// -// yetAnotherArrayOfSongs.push(filePath); -// } -// } -// } -// } - -// -// function runOnStart(){ -// -// // Loop through users -// // Scan one at a time -// -// // Check DB for the last addition timed -// -// // Loop through users files -// -// // Check for modification time -// // if modification time is newer than the latest time, go to next step -// // if file exists inthe db already, add it to modificationCheckArray -// // if it doesn't exist, add it to newSongArray -// -// } - - - -exports.setup = function(mstream, users, db){ - const rootDirCopy = rootDir; - dbCopy = db; - - runOnStart(); - - // scan and screate database - mstream.get('/db/recursive-scan-mstream', function(req,res){ - - // Check if this is already running - if(scanLock === true){ - // Return error - res.status(401).send('{"error":"Scan in progress"}'); - return; - } - - try{ - // turn on scan lock - scanLock = true; - - // Make sure directory exits - var fileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; - - countFiles(rootDirCopy, fileTypesArray); - totalFileCount = yetAnotherArrayOfSongs.length; - - // TODO: Move this - // dbCopy.serialize(function() { - // // These two queries will run sequentially. - // dbCopy.run("drop table if exists items;"); - // dbCopy.run("CREATE TABLE items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { - // // These queries will run in parallel and the second query will probably - // // fail because the table might not exist yet. - // console.log('TABLES CREATED'); - // - // parse = parseAllFiles(); - // parse.next(); - // }); - // }); - - }catch(err){ - // Remove lock - scanLock = false; - - // TODO Log error - // console.log(err); - return; - } - - res.send("YA DID IT"); - - }); - - - - mstream.get('/db/status-mstream', function(req, res){ - var returnObject = {}; - - returnObject.locked = scanLock; - - - if(scanLock){ - - returnObject.totalFileCount = totalFileCount; - returnObject.filesLeft = yetAnotherArrayOfSongs.length; - - res.json(returnObject); - - }else{ - var sql = 'SELECT Count(*) FROM items'; - - dbCopy.get(sql, function(err, row){ - if(err){ - console.log(err.message); - - res.status(500).json({ error: err.message }); - return; - } - - - var fileCountDB = row['Count(*)']; // TODO: Is this correct??? - - returnObject.totalFileCount = fileCountDB; - res.json(returnObject); - - }); - } - - }); - +try{ + if(fe.extname(process.argv[process.argv.length-1]) == '.json' && fs.statSync(process.argv[process.argv.length-1]).isFile()){ + var loadJson = JSON.parse(fs.readFileSync(args[args.length-1], 'utf8')); + }else{ + console.log('Bad input'); + process.exit(); + } +}catch(error){ + console.log('JSON file does not appear to exist'); + process.exit(); } - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// TODO: Call Function // 2.0 diff --git a/modules/db-management/database-master.js b/modules/db-management/database-master.js index 3adad60..6834415 100644 --- a/modules/db-management/database-master.js +++ b/modules/db-management/database-master.js @@ -1,6 +1,7 @@ -const spawn = require('child_process').spawn; exports.setup = function(mstream, users, publicDBType){ + const spawn = require('child_process').spawn; + // sqlite3, mysql, sequelize, lokiJS, (? posgres) // This will allow us to make sqlite3 an optional dependancy once lokiJS works @@ -22,6 +23,7 @@ exports.setup = function(mstream, users, publicDBType){ mstream.get('/db/recursive-scan', function(req,res){ // Get user's db setup + var userDB = req.user.db; // TODO: declare this in main file // spawn a child_process to scan // spawnBeets(user, userCommand); // For beets we need pull the exact command to launch from the user config @@ -29,15 +31,42 @@ exports.setup = function(mstream, users, publicDBType){ }); function spawnDefault(user){ + // TODO: Fix This + // Send in DB config + // Send in user + const ls = spawn('ls', ['-lh', '/usr']); + ls.stdout.on('data', (data) => { + console.log(`stdout: ${data}`); + }); + + ls.stderr.on('data', (data) => { + console.log(`stderr: ${data}`); + }); + + ls.on('close', (code) => { + console.log(`child process exited with code ${code}`); + }); } + // TODO: Special function that just transfers fiels from users private DB to public DB + mstream.get('/db/import-DB', function(req,res){ + // Get user info + // Pull user's private DB config + // Return if user is not using private DB + // Delete users files + // Pull all files from DB and add to publicDB + }); + + // TODO: Handle user status mstream.get('/db/status-mstream', function(req, res){ }); + +// TO BE REMOVED // ======================================================================================== // // Either copy from sqliteDB or use built-in functions // // Go through all vars and determine plugins needed diff --git a/mstream.js b/mstream.js index e761140..e3b4e52 100755 --- a/mstream.js +++ b/mstream.js @@ -462,106 +462,104 @@ function getFileType(filename){ -// TODO: Save playlist according to user and the user's music DIR -mstream.post('/saveplaylist', function (req, res){ - var title = req.body.title; - var songs = req.body.stuff; - - // Check if this playlist already exists - // TODO: Add field for username - db.all("SELECT id FROM mstream_playlists WHERE playlist_name = ?;", title, function(err, rows) { - - db.serialize(function() { - - // We need to delete anys existing entries - if(rows && rows.length > 0){ - db.run("DELETE FROM mstream_playlists WHERE playlist_name = ?;", title); - } - - // Now we add the new entries - var sql2 = "insert into mstream_playlists (playlist_name, filepath) values "; - var sqlParser = []; - - while(songs.length > 0) { - var song = songs.shift(); - - sql2 += "(?, ?), "; - sqlParser.push(title); - sqlParser.push( fe.join(req.user.musicDir, song) ); // TODO: User music dir - } - - sql2 = sql2.slice(0, -2); - sql2 += ";"; - - db.run(sql2, sqlParser, function(){ - res.send('DONE'); - }); - - }); - }); -}); - - -mstream.get('/getallplaylists', function (req, res){ - - // TODO: In V2 we need to change this to ignore hidden playlists - // TODO: db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE hide=0;", function(err, rows){ - db.all("SELECT DISTINCT playlist_name FROM mstream_playlists", function(err, rows){ - var playlists = []; - - // loop through files - for (var i = 0; i < rows.length; i++) { - if(rows[i].playlist_name){ - playlists.push({name: rows[i].playlist_name}); - } - } - - res.send(JSON.stringify(playlists)); - }); -}); - -mstream.get('/loadplaylist', function (req, res){ - var playlist = req.query.playlistname; - - db.all("SELECT * FROM mstream_playlists WHERE playlist_name = ? ORDER BY id COLLATE NOCASE ASC", playlist, function(err, rows){ - var returnThis = []; - - for (var i = 0; i < rows.length; i++) { - - // var tempName = rows[i].filepath.split('/').slice(-1)[0]; - var tempName = fe.basename(rows[i].filepath); - var extension = getFileType(rows[i].filepath); - var filepath = slash(fe.relative(req.user.musicDir, rows[i].filepath)); // TODO - - returnThis.push({name: tempName, file: filepath, filetype: extension }); - } - - res.send(JSON.stringify(returnThis)); - }); - -}); - - -mstream.get('/deleteplaylist', function(req, res){ - var playlistname = req.query.playlistname; - - // Handle a soft delete - if(req.query.hide && parseInt(req.query.hide) === 1 ){ - db.run("UPDATE mstream_playlists SET hide = 1 WHERE playlist_name = ?;", playlistname, function(){ - res.send('DONE'); - - }); - }else{ // Permentaly delete - - // Delete playlist from DB - db.run("DELETE FROM mstream_playlists WHERE playlist_name = ?;", playlistname, function(){ - res.send('DONE'); - - }); - } - - -}); +// // TODO: Save playlist according to user and the user's music DIR +// mstream.post('/saveplaylist', function (req, res){ +// var title = req.body.title; +// var songs = req.body.stuff; +// +// // Check if this playlist already exists +// // TODO: Add field for username +// db.all("SELECT id FROM mstream_playlists WHERE playlist_name = ?;", title, function(err, rows) { +// +// db.serialize(function() { +// +// // We need to delete anys existing entries +// if(rows && rows.length > 0){ +// db.run("DELETE FROM mstream_playlists WHERE playlist_name = ?;", title); +// } +// +// // Now we add the new entries +// var sql2 = "insert into mstream_playlists (playlist_name, filepath) values "; +// var sqlParser = []; +// +// while(songs.length > 0) { +// var song = songs.shift(); +// +// sql2 += "(?, ?), "; +// sqlParser.push(title); +// sqlParser.push( fe.join(req.user.musicDir, song) ); // TODO: User music dir +// } +// +// sql2 = sql2.slice(0, -2); +// sql2 += ";"; +// +// db.run(sql2, sqlParser, function(){ +// res.send('DONE'); +// }); +// +// }); +// }); +// }); +// +// +// mstream.get('/getallplaylists', function (req, res){ +// +// // TODO: In V2 we need to change this to ignore hidden playlists +// // TODO: db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE hide=0;", function(err, rows){ +// db.all("SELECT DISTINCT playlist_name FROM mstream_playlists", function(err, rows){ +// var playlists = []; +// +// // loop through files +// for (var i = 0; i < rows.length; i++) { +// if(rows[i].playlist_name){ +// playlists.push({name: rows[i].playlist_name}); +// } +// } +// +// res.send(JSON.stringify(playlists)); +// }); +// }); +// +// mstream.get('/loadplaylist', function (req, res){ +// var playlist = req.query.playlistname; +// +// db.all("SELECT * FROM mstream_playlists WHERE playlist_name = ? ORDER BY id COLLATE NOCASE ASC", playlist, function(err, rows){ +// var returnThis = []; +// +// for (var i = 0; i < rows.length; i++) { +// +// // var tempName = rows[i].filepath.split('/').slice(-1)[0]; +// var tempName = fe.basename(rows[i].filepath); +// var extension = getFileType(rows[i].filepath); +// var filepath = slash(fe.relative(req.user.musicDir, rows[i].filepath)); // TODO +// +// returnThis.push({name: tempName, file: filepath, filetype: extension }); +// } +// +// res.send(JSON.stringify(returnThis)); +// }); +// +// }); +// +// +// mstream.get('/deleteplaylist', function(req, res){ +// var playlistname = req.query.playlistname; +// +// // Handle a soft delete +// if(req.query.hide && parseInt(req.query.hide) === 1 ){ +// db.run("UPDATE mstream_playlists SET hide = 1 WHERE playlist_name = ?;", playlistname, function(){ +// res.send('DONE'); +// +// }); +// }else{ // Permentaly delete +// +// // Delete playlist from DB +// db.run("DELETE FROM mstream_playlists WHERE playlist_name = ?;", playlistname, function(){ +// res.send('DONE'); +// +// }); +// } +// }); // Download a zip file of music @@ -618,49 +616,49 @@ const mstreamDB = require('./modules/db-management/database-master.js'); mstreamDB.setup(mstream, program.users, publicDBType); -mstream.post('/db/search', function(req, res){ - var searchTerm = "%" + req.body.search + "%" ; - - var returnThis = {"albums":[], "artists":[]}; - - // TODO: Combine SQL calls into one - db.serialize(function() { - - var sqlAlbum = "SELECT DISTINCT album FROM items WHERE items.album LIKE ? ORDER BY album COLLATE NOCASE ASC;"; - db.all(sqlAlbum, searchTerm, function(err, rows) { - if(err){ - res.status(500).json({ error: 'DB Error' }); - return; - } - - for (var i = 0; i < rows.length; i++) { - if(rows[i].album){ - // rows.splice(i, 1); - returnThis.albums.push(rows[i].album); - } - } - }); - - - var sqlAlbum = "SELECT DISTINCT artist FROM items WHERE items.artist LIKE ? ORDER BY artist COLLATE NOCASE ASC;"; - db.all(sqlAlbum, searchTerm, function(err, rows) { - if(err){ - res.status(500).json({ error: 'DB Error' }); - return; - } - - for (var i = 0; i < rows.length; i++) { - if(rows[i].artist){ - // rows.splice(i, 1); - returnThis.artists.push(rows[i].artist); - } - } - - res.send(JSON.stringify(returnThis)); - - }); - }); -}); +// mstream.post('/db/search', function(req, res){ +// var searchTerm = "%" + req.body.search + "%" ; +// +// var returnThis = {"albums":[], "artists":[]}; +// +// // TODO: Combine SQL calls into one +// db.serialize(function() { +// +// var sqlAlbum = "SELECT DISTINCT album FROM items WHERE items.album LIKE ? ORDER BY album COLLATE NOCASE ASC;"; +// db.all(sqlAlbum, searchTerm, function(err, rows) { +// if(err){ +// res.status(500).json({ error: 'DB Error' }); +// return; +// } +// +// for (var i = 0; i < rows.length; i++) { +// if(rows[i].album){ +// // rows.splice(i, 1); +// returnThis.albums.push(rows[i].album); +// } +// } +// }); +// +// +// var sqlAlbum = "SELECT DISTINCT artist FROM items WHERE items.artist LIKE ? ORDER BY artist COLLATE NOCASE ASC;"; +// db.all(sqlAlbum, searchTerm, function(err, rows) { +// if(err){ +// res.status(500).json({ error: 'DB Error' }); +// return; +// } +// +// for (var i = 0; i < rows.length; i++) { +// if(rows[i].artist){ +// // rows.splice(i, 1); +// returnThis.artists.push(rows[i].artist); +// } +// } +// +// res.send(JSON.stringify(returnThis)); +// +// }); +// }); +// }); From 3d8f64f252d74f65cdd248ea275c980290b6ef87 Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Mon, 28 Nov 2016 22:13:31 -0500 Subject: [PATCH 04/11] The server portion works. Need to finish the database building and import tools --- modules/configure-commander.js | 11 +- modules/configure-json-file.js | 21 +- .../db-management/database-beets-manager.js | 38 +- .../db-management/database-default-manager.js | 30 +- modules/db-management/database-master.js | 124 ++-- modules/db-read/database-public-sqlite.js | 13 +- modules/db-write/database-default-sqlite.js | 5 + mstream.js | 622 ++++-------------- package.json | 1 - public/js/mstream.js | 12 + untitled folder/JSONsqlite.json | 31 + untitled folder/exampleNewJSON.json | 25 +- 12 files changed, 322 insertions(+), 611 deletions(-) create mode 100644 untitled folder/JSONsqlite.json diff --git a/modules/configure-commander.js b/modules/configure-commander.js index 1373df9..5074c9d 100644 --- a/modules/configure-commander.js +++ b/modules/configure-commander.js @@ -1,22 +1,25 @@ exports.setup = function(args){ + // TODO: This exists to have a simple setup from the command line + // Removed beets support from cli launch + // Setup Command Line Interface var program = require('commander'); program - .version('1.21.0') + .version('1.30.0') .option('-p, --port ', 'Select Port', /^\d+$/i, 3000) .option('-t, --tunnel', 'Use nat-pmp to configure port fowarding') .option('-g, --gateip ', 'Manually set gateway IP for the tunnel option') .option('-u, --user ', 'Set Username') .option('-x, --password ', 'Set Password') - .option('-e, --email ', 'Set User Email (optional)') + // .option('-e, --email ', 'Set User Email (optional)') .option('-G, --guest ', 'Set Guest Username') .option('-X, --guestpassword ', 'Set Guest Password') // .option('-k, --key ', 'Add SSL Key') // .option('-c, --cert ', 'Add SSL Certificate') .option('-d, --database ', 'Specify Database Filepath', 'mstreamdb.lite') - .option('-b, --beetspath ', 'Specify Folder where Beets DB should import music from. This also overides the normal DB functions with functions that integrate with beets DB') - .option('-b, --databaseplugin ', '', /^(default|beets)$/i, 'default') + // .option('-b, --beetspath ', 'Specify Folder where Beets DB should import music from. This also overides the normal DB functions with functions that integrate with beets DB') + // .option('-b, --databaseplugin ', '', /^(default|beets)$/i, 'default') .option('-i, --userinterface ', 'Specify folder name that will be served as the UI', 'public') .option('-f, --filepath ', 'Set the path of your music directory', process.cwd()) .option('-s, --secret ', 'Set the login secret key') diff --git a/modules/configure-json-file.js b/modules/configure-json-file.js index 3f5954d..0849113 100644 --- a/modules/configure-json-file.js +++ b/modules/configure-json-file.js @@ -10,24 +10,17 @@ exports.setup = function(args){ return false; } - // Check for validity - if(!loadJson.user){ - loadJson.user = 'admin'; + + if(!loadJson.database_plugin){ + console.log('Please Configure DB'); + return false; } - if(!loadJson.databaseplugin){ - loadJson.databaseplugin = 'default'; - }else{ - var re = new RegExp("^(default|beets)$"); - if(!re.test(loadJson.databaseplugin)){ - console.log('Incorrect database plugin. Please update and try again'); - return false; - } + if(!loadJson.userinterface){ + loadJson.userinterface = "public"; } - if(!loadJson.filepath){ - loadJson.filepath = process.cwd(); - } + // Export JSON return loadJson; diff --git a/modules/db-management/database-beets-manager.js b/modules/db-management/database-beets-manager.js index 77a70c7..a2291f5 100644 --- a/modules/db-management/database-beets-manager.js +++ b/modules/db-management/database-beets-manager.js @@ -1,3 +1,39 @@ +// Special functions for beets DB + +// Download the database +// TODO: Fix these +mstream.get('/db/download-db', function(req, res){ + var file = program.database; + + res.download(file); // Set disposition and send it. +}); + + +// Get hash of database +mstream.get( '/db/hash', function(req, res){ + var hash = crypto.createHash('sha256'); + var fileStream = fs.createReadStream(program.database); + + hash.setEncoding('hex'); + fileStream.pipe(hash, { end: false }); + + + fileStream.on('end', function () { + hash.end(); + + var returnThis = { + hash:String(hash.read()) + }; + + res.send(JSON.stringify(returnThis)); + + }); +}); + + + + + // // TODO: This thing has to be tested // // // TODO: These functions are for interacting withe the beets DB @@ -12,7 +48,7 @@ // exports.setup = function(mstream, program, rootDir, db){ // const scanThisDir = program.beetspath; // TODO: Check that this is a real directory // -// +// // mstream.get('/db/recursive-scan-beets', function(req,res){ // // if(scanLock === true){ diff --git a/modules/db-management/database-default-manager.js b/modules/db-management/database-default-manager.js index 92e8a9e..7c80302 100644 --- a/modules/db-management/database-default-manager.js +++ b/modules/db-management/database-default-manager.js @@ -27,6 +27,9 @@ try{ process.exit(); } +// TODO: Check JSON for nencessary info + + // TODO: Call Function @@ -41,19 +44,23 @@ var arrayOfSongs; // Holds songs for DB to process // TODO: Move out of global s var arrayOfScannedFiles = []; // Holds files for from recursive scan var parseFilesGenerator; // This Generator is used in two places. Should it be seperated? -var scanDirLock = false; +// var scanDirLock = false; //TODO: Pull in correct module -const dbRead = require('./modules/db-write/database-default-sqlite.js') + +const dbRead = require('./modules/db-write/database-default-'+loadJson.dbType+'.js'); +if(loadJson.dbType == 'sqlite'){ + dbRead.setup(loadJson.dbSettings.path); // TODO: Pass this in +} function rescanAllDirectoriesWrapper(){ - if(scanDirLock === true){ - // TODO: If scanlock == true, aleart user to try again once scanning is done - // TODO: Enable Button - return; - } + // if(scanDirLock === true){ + // // TODO: If scanlock == true, aleart user to try again once scanning is done + // // TODO: Enable Button + // return; + // } - scanDirLock = true; + // scanDirLock = true; // TODO: Disable Button parseFilesGenerator = rescanAllDirectories(dir); @@ -80,7 +87,7 @@ function *rescanAllDirectories(directoryToScan){ } // Re-enable scanning - scanDirLock = false; + // scanDirLock = false; } @@ -275,3 +282,8 @@ function recursiveScan(dir, fileTypesArray){ } } } + +// TODO: +function insertEntries(){ + dbRead.sendUserFiles(); +} diff --git a/modules/db-management/database-master.js b/modules/db-management/database-master.js index 6834415..a2ba3f6 100644 --- a/modules/db-management/database-master.js +++ b/modules/db-management/database-master.js @@ -1,52 +1,88 @@ - -exports.setup = function(mstream, users, publicDBType){ - const spawn = require('child_process').spawn; +//exports.setup = function(mstream, users, publicDBType, dbSettings){ +exports.setup = function(mstream, program){ + const child = require('child_process'); - // sqlite3, mysql, sequelize, lokiJS, (? posgres) - // This will allow us to make sqlite3 an optional dependancy once lokiJS works - // Make sure all users have uniform settings for the inital run, we'll handle mixed settings later - - - - - - // Load the public DB plugin - // sqlite3 and mysql to start, add lokiJS support latet (and maybe posgres later), maybe a NO DB option even later + // Load the public DB plugin + // sqlite3 and mysql to start, add lokiJS support latet (and maybe posgres later), maybe a NO DB option even later // The following api calls are all handled on a public level // They can be moved into a public plugin // pull plugin from masterDBType - const mstreamReadPublicDB = require('./modules/db-read/database-public-'+publicDBType+'.js'); - mstreamReadPublicDB.setup(mstream); + const mstreamReadPublicDB = require('../db-read/database-public-'+program.database_plugin.type+'.js'); + mstreamReadPublicDB.setup(mstream, program.database_plugin); + var userDBStatus = {}; + mstream.get('/db/recursive-scan', function(req,res){ - // Get user's db setup - var userDB = req.user.db; // TODO: declare this in main file + // Check if user is already being scanned + if(userDBStatus[req.user.username] == true){ + res.send('In Process. Please check status.'); + return; + } + // + userDBStatus[req.user.username] = true; + + // Get user's db setup + if(!req.user.privateDB || req.user.privateDB == 'DEFAULT'){ + forkDefault(req.user, program.database_plugin); + res.send('IT\'S HAPPENING!'); + return; + } + + if(req.user.privateDB == 'BEETS'){ + forkBeets(req.user); + res.send('IT\'S HAPPENING! \n NOW WITH 60% MORE BEETS!'); + return; + } + + // YOUR CONFIG IS BAD AND YOU SHOULD FEEL BAD + userDBStatus[req.user.username] = false; + res.send('YOUR CONFIG IS BAD AND YOU SHOULD FEEL BAD. ABORTING!'); - // spawn a child_process to scan - // spawnBeets(user, userCommand); // For beets we need pull the exact command to launch from the user config - // spawnDefault(user); }); - function spawnDefault(user){ - // TODO: Fix This - // Send in DB config - // Send in user - const ls = spawn('ls', ['-lh', '/usr']); - ls.stdout.on('data', (data) => { + /////////////////////////// + // TODO: Should we have a API call that can kill any process associated with a user and reset their scan value to false? + /////////////////////////// + + /////////////////////////// + // TODO: We could use some kind of manager to make sure we don't spawn to many child processes + // For know we spawn indiscriminately and let the CPU sort it out + /////////////////////////// + + // TODO: Fill this out + function forkBeets(user, publicDBType, dbSettings){ + + } + + function forkDefault(user, dbSettings){ + // TODO: IMPLEMENT FORK PROPERLY + // SEND JSON DATA TO WORKER PROCESS + var jsonLoad = { + username:user.username, + userDir:user.musicDir, + dbSettings:dbSettings + } + + const forkedScan = child.fork(__dirname + '/modules/db-management/database-default-manager.js', [JSON.stringify(jsonLoad)]); + + forkedScan.stdout.on('data', (data) => { console.log(`stdout: ${data}`); }); - ls.stderr.on('data', (data) => { + forkedScan.stderr.on('data', (data) => { console.log(`stderr: ${data}`); }); - ls.on('close', (code) => { + forkedScan.on('close', (code) => { + userDBStatus[user.username] = false; console.log(`child process exited with code ${code}`); }); + + // TODO: Need to make an on error } @@ -66,27 +102,6 @@ exports.setup = function(mstream, users, publicDBType){ -// TO BE REMOVED -// ======================================================================================== -// // Either copy from sqliteDB or use built-in functions -// // Go through all vars and determine plugins needed -// var useMstream = false; -// var useBeets = false; -// -// for (i = 0; i < program.users.length; i++) { -// // Check for beets -// // var useMstream = true; -// // var useBeets = true; -// } -// if(useMstream){ -// const mstreamWritePublicDB = require('./modules/database-default-'+publicDBType+'.js'); // FIXME; Rename -// mstreamWritePublicDB.setup(mstream, users, db); -// } -// if(useBeets){ -// const beetsWritePublicDB = require('./modules/database-beets-'+publicDBType+'.js'); // FIXME; Rename -// } -// ======================================================================================== - @@ -105,14 +120,3 @@ exports.setup = function(mstream, users, publicDBType){ // }); } - - - -// Case 1: totalally managed by mstream, sqlite3 for everything - -// Case 2: beets DB for every user, mysql for public DB - -// Case 3: totally managed by mstream, lokiJS for everything - -// Next step: mixed user settings -// Next step: backup as local DB and import from backup diff --git a/modules/db-read/database-public-sqlite.js b/modules/db-read/database-public-sqlite.js index 889a17e..5eaa975 100644 --- a/modules/db-read/database-public-sqlite.js +++ b/modules/db-read/database-public-sqlite.js @@ -1,3 +1,6 @@ +const sqlite3 = require('sqlite3').verbose(); + + // function that takes in a json array of songs and saves them to the sqlite db // must contain the username and filepath for each song @@ -6,7 +9,6 @@ // function that takes ina playlsit name and searchs db for that playlist and returns a json array of songs for that playlist // BASICALLY, all the functions we have no but de-couple them from the Express API calls -// TODO: Setup SQLite function getFileType(filename){ @@ -14,13 +16,14 @@ function getFileType(filename){ } -exports.setup = function(mstream){ - db.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER, user VARCHAR);", function() { - // console.log('TABLES CREATED'); +exports.setup = function(mstream, dbSettings){ + const db = new sqlite3.Database(dbSettings.dbPath); + + // Setup DB + db.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER, user VARCHAR, filesize INTEGER, file_created_date INTEGER, file_modified_date INTEGER);", function() { }); // Create a playlist table db.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, user VARCHAR, created datetime default current_timestamp);", function() { - // console.log('PLAYLIST TABLE CREATED'); }); diff --git a/modules/db-write/database-default-sqlite.js b/modules/db-write/database-default-sqlite.js index 375c0e9..99a1501 100644 --- a/modules/db-write/database-default-sqlite.js +++ b/modules/db-write/database-default-sqlite.js @@ -1,6 +1,11 @@ // functions that store data into the SQLite DB // These functions will take in JSON arrays of song data and then save that dat to the DB +const sqlite3 = require('sqlite3').verbose(); +var db; +exports.setup = function(dbPath){ + db = new sqlite3.Database(dbPath); +} exports.getUserFiles = function(user, callback){ db.all("SELECT path, file_modified_date FROM files WHERE user=? ;" [thisUser], function(err, rows){ diff --git a/mstream.js b/mstream.js index e3b4e52..fb9ef08 100755 --- a/mstream.js +++ b/mstream.js @@ -3,19 +3,18 @@ const express = require('express'); const mstream = express(); -const fs = require('graceful-fs'); // File System +const fs = require('fs'); // File System const fe = require('path'); const bodyParser = require('body-parser'); const archiver = require('archiver'); // Zip Compression const os = require('os'); const crypto = require('crypto'); const slash = require('slash'); -// const sqlite3 = require('sqlite3').verbose(); -var startup = 'configure-commander'; // If the user gives a json file then try pulling the config from that try{ + var startup = 'configure-commander'; if(fe.extname(process.argv[process.argv.length-1]) == '.json' && fs.statSync(process.argv[process.argv.length-1]).isFile()){ startup = 'configure-json-file'; } @@ -28,20 +27,6 @@ const program = require('./modules/' + startup + '.js').setup(process.argv); if(program == false){ process.exit(); } -// const db = new sqlite3.Database(program.database); - - -// TODO: Move these to db modules -// // If we are not using Beets DB, we need to prep the DB -// if(program.databaseplugin === 'default'){ -// db.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { -// // console.log('TABLES CREATED'); -// }); -// } -// // Create a playlist table -// db.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, created datetime default current_timestamp);", function() { -// // console.log('PLAYLIST TABLE CREATED'); -// }); @@ -49,8 +34,9 @@ if(program == false){ // Normalize for all OS // Make sure it's a directory // Loop through and makeure all user Dirs are real +// TODO: Move all checks to the JSON module if(program.users){ - for (i = 0; i < program.users.length; i++) { + for (let i = 0; i < program.users.length; i++) { //TODO: Check usernames for forbidden chars // TODO: Assure all usernames are unique @@ -58,73 +44,24 @@ if(program.users){ // TODO: Assure only one user per filepath - // TODO: Assure all users are usingthe same DB schema - if(!fs.statSync( program.users[i].musicDir ).isDirectory()){ console.log(program.users[i].username + " music directory could not be found"); process.exit(); } } -}else if(!fs.statSync( program.filepath ).isDirectory()){ - console.log('GIVEN DIRECTORY DOES NOT APPEAR TO BE REAL'); - process.exit(); } -if(program.user && program.password){ - // Move program.username and program.password to program.users - var newUser = { - "username":program.user, - "password":program.password, - "musicDir":program.filepath - }; - - if(program.email){ - newUser.email = program.email - } - - // TODO: Handle Guest Account - // if(program.guest && program.guestPassword){ - - // } - - program.users.push(newUser); -} - - -// Check that this is a real dir -if(!fs.statSync( fe.join(__dirname, program.userinterface) ).isDirectory()){ - console.log('The userinterface was not found. Closing...'); - process.exit(); -} - -// Static files -// TODO: Loop through and create sperate virtual paths for all user dirs -mstream.use( express.static(fe.join(__dirname, program.userinterface) )); -if(program.users){ - for (i = 0; i < program.users.length; i++) { - // TODO: Check if musicDir is real - - mstream.use( '/' + program.users[i].username + '/' , express.static( program.users[i].musicDir )); - } -}else{ - var rootDir = fe.normalize(program.filepath); - // Normalize It - if(!fe.isAbsolute(program.filepath) ){ - rootDir = fe.join(process.cwd, rootDir); - } - mstream.use( '/' , express.static( rootDir )); -} // Magic Middleware Things mstream.use(bodyParser.json()); // support json encoded bodies mstream.use(bodyParser.urlencoded({ extended: true })); // support encoded bodies -// Handle ports. Default is 3000 -const port = program.port; -console.log('Access mStream locally: http://localhost:' + port); +// Print the local network IP +console.log('Access mStream locally: http://localhost:' + program.port); +console.log('Access mStream on your local network: http://' + require('my-local-ip')() + ':' + program.port); // Handle Port Forwarding @@ -133,13 +70,17 @@ console.log('Access mStream locally: http://localhost:' + port); if(program.tunnel){ const tunnel = require('./modules/auto-port-forwarding.js'); tunnel.tunnel_uPNP(program.port); - tunnel.logUrl(port); + tunnel.logUrl(program.port); } -// Print the local network IP -console.log('Access mStream on your local network: http://' + require('my-local-ip')() + ':' + port); +// Check that this is a real dir +if(!fs.statSync( fe.join(__dirname, program.userinterface) ).isDirectory()){ + console.log('The userinterface was not found. Closing...'); + process.exit(); +} +mstream.use( express.static(fe.join(__dirname, program.userinterface) )); // Serve the webapp mstream.get('/', function (req, res) { @@ -150,45 +91,12 @@ mstream.get('/', function (req, res) { // Login functionality if(program.users){ - - // TODO: password change function - if(program.email){ - mstream.post('/change-password-request', function (req, res) { - // Get email address from request - // validate email against user array - - // Generate change password token - - // Invalidate all other change password tokens - - // Email the user the token - - res.sendFile( 'COMING SOON!' ); - }); - - - // TODO: Add New user - // Check for root user and password - // Add credentials to user array - - mstream.post('/change-password', function (req, res){ - // Check token - - // Get new password - - // Hash password and update user array - - res.sendFile( 'COMING SOON!' ); - }); - } - // Use bcrypt for password storage const bcrypt = require('bcrypt'); const jwt = require('jsonwebtoken'); // used to create, sign, and verify tokens var secret; var secretIsFile = false; - // Check for filepath try{ if(fs.statSync(program.secret).isFile()){ @@ -196,64 +104,87 @@ if(program.users){ } }catch(error){} - if(secretIsFile === true){ + // If the given secret is a filepath secret = fs.readFileSync(program.secret, 'utf8'); }else if(program.secret){ + // Otherwise just use secret as is secret = String(program.secret); }else{ + // If no secret was given, generate one require('crypto').randomBytes(48, function(err, buffer) { secret = buffer.toString('hex'); }); } + + // TODO: Add New user functionality + // Check for root user and password + // Add credentials to user array + + // TODO: Need a way to store and use already hashed passwords + + + // TODO: password change function + if(false == true){ + mstream.post('/change-password-request', function (req, res) { + // Get email address from request + // validate email against user array + // Generate change password token + // Invalidate all other change password tokens + // Email the user the token + + res.sendFile( 'COMING SOON!' ); + }); + + mstream.post('/change-password', function (req, res){ + // Check token + // Get new password + // Hash password and update user array + + res.sendFile( 'COMING SOON!' ); + }); + } + + // Create the user array var Users = {}; - - // TODO: Construct user array - for (i = 0; i < program.users.length; i++) { + // Construct user array + for (let i = 0; i < program.users.length; i++) { Users[program.users[i].username] = { - "musicDir":program.users[i].musicDir, + musicDir:program.users[i].musicDir, } if(program.users[i].email){ Users[program.users[i].username].email = program.users[i].email; } - } - // Users[program.user] = { - // "guest": false, - // "guestPassword":"", - // "password":'', - // "email":"", - // "musicDir":"", - // - // } - - // TODO: Break salt generation and password management into a loop and seperate function - // Encrypt the password - bcrypt.genSalt(10, function(err, salt) { - bcrypt.hash(program.password, salt, function(err, hash) { - // Store hash in your password DB. - Users[program.user]['password'] = hash; - }); - }); - // Handle guest account - if(program.guest && program.guestpassword){ - Users[program.guest] = { - 'guest': true, - 'password':'', + if(program.users[i].privateDB){ + Users[program.users[i].username].email = program.users[i].privateDB; } - // Encrypt the password + if(program.users[i].privateDBOptions){ + Users[program.users[i].username].email = program.users[i].privateDBOptions; + } + + generateSaltedPassword(program.users[i].username, program.users[i].password); + + //////////////////////////////// + // TODO: Handle Guest Options // + //////////////////////////////// + + // TODO: We could use a better way of mapping users to vPaths + mstream.use( '/' + program.users[i].username + '/' , express.static( program.users[i].musicDir )); + } + + + function generateSaltedPassword(username, password){ bcrypt.genSalt(10, function(err, salt) { - bcrypt.hash(program.guestpassword, salt, function(err, hash) { + bcrypt.hash(password, salt, function(err, hash) { // Store hash in your password DB. - Users[program.guest]['password'] = hash; + Users[username]['password'] = hash; }); }); } - - // Failed Login Attempt mstream.get('/login-failed', function (req, res) { // Wait before sending the response @@ -291,17 +222,15 @@ if(program.users){ var user = Users[username]; user['username'] = username; - // Make a token for the user - var token = jwt.sign(user, secret); - // return the information including token as JSON var sendThis = { success: true, message: 'Welcome To mStream', - token: token }; + vPath: user.username, + token: jwt.sign(user, secret) // Make the token + }; res.send(JSON.stringify(sendThis)); - }); }); @@ -331,38 +260,41 @@ if(program.users){ // Set user request data req.user = decoded; - - next(); }); }); + // TODO: Middleware that prevents users from accessing another users files - // TODO: Authenticate all HTTP requests for music files (mp3 and other formats) }else{ // Dummy data mstream.use(function(req, res, next) { req.user = { - "username":"", - "musicDir":program.filepath + username:"", + musicDir:program.filepath }; next(); }); + mstream.use( '/' , express.static( program.users[i].musicDir )); } + + + + // Test function // Used to determine the user has a working login token -// TODO: This will return the virtual file path directory needed to access msuci files mstream.get('/ping', function(req, res){ + // TODO: Guest status var returnObject = { - 'vPath' = req.user.username, - 'guest' = false, // TODO: return guest status + vPath: req.user.username, + guest: false }; - res.send(JSON.stringify(returnObject); + res.send(JSON.stringify(returnObject)); }); @@ -406,52 +338,42 @@ mstream.post('/dirparser', function (req, res) { // loop through files for (var i=0; i < files.length; i++) { - var tempDirArray = {}; - var tempFileArray = {}; - var filePath = fe.join(path, files[i]); try{ - var stat = fs.statSync(filePath); + var stat = fs.statSync(fe.join(path, files[i])); }catch(error){ // Bad file, ignore and continue // TODO: Log This continue; } - // Handle Directories if(stat.isDirectory()){ - tempDirArray["type"] = 'directory'; - tempDirArray["name"] = files[i]; - - directories.push(tempDirArray); + directories.push({ + type:"directory", + name:files[i] + }); }else{ // Handle Files - - // Get the file extension var extension = getFileType(files[i]); - - if (fileTypesArray.indexOf(extension) > -1 && masterFileTypesArray.indexOf(extension) > -1) { - tempFileArray["type"] = extension; - tempFileArray["name"] = files[i]; - - filesArray.push(tempFileArray); + filesArray.push({ + type:extension, + name:files[i] + }); } } - } - // TODO: rootdir stuff here var returnPath = slash( fe.relative(req.user.musicDir, path) ); if(returnPath.slice(-1) !== '/'){ returnPath += '/'; } + // Combine list of directories and mp3s var finalArray = { path:returnPath, contents:filesArray.concat(directories)}; // Send back some JSON res.send(JSON.stringify(finalArray)); - }); @@ -461,120 +383,17 @@ function getFileType(filename){ } - -// // TODO: Save playlist according to user and the user's music DIR -// mstream.post('/saveplaylist', function (req, res){ -// var title = req.body.title; -// var songs = req.body.stuff; -// -// // Check if this playlist already exists -// // TODO: Add field for username -// db.all("SELECT id FROM mstream_playlists WHERE playlist_name = ?;", title, function(err, rows) { -// -// db.serialize(function() { -// -// // We need to delete anys existing entries -// if(rows && rows.length > 0){ -// db.run("DELETE FROM mstream_playlists WHERE playlist_name = ?;", title); -// } -// -// // Now we add the new entries -// var sql2 = "insert into mstream_playlists (playlist_name, filepath) values "; -// var sqlParser = []; -// -// while(songs.length > 0) { -// var song = songs.shift(); -// -// sql2 += "(?, ?), "; -// sqlParser.push(title); -// sqlParser.push( fe.join(req.user.musicDir, song) ); // TODO: User music dir -// } -// -// sql2 = sql2.slice(0, -2); -// sql2 += ";"; -// -// db.run(sql2, sqlParser, function(){ -// res.send('DONE'); -// }); -// -// }); -// }); -// }); -// -// -// mstream.get('/getallplaylists', function (req, res){ -// -// // TODO: In V2 we need to change this to ignore hidden playlists -// // TODO: db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE hide=0;", function(err, rows){ -// db.all("SELECT DISTINCT playlist_name FROM mstream_playlists", function(err, rows){ -// var playlists = []; -// -// // loop through files -// for (var i = 0; i < rows.length; i++) { -// if(rows[i].playlist_name){ -// playlists.push({name: rows[i].playlist_name}); -// } -// } -// -// res.send(JSON.stringify(playlists)); -// }); -// }); -// -// mstream.get('/loadplaylist', function (req, res){ -// var playlist = req.query.playlistname; -// -// db.all("SELECT * FROM mstream_playlists WHERE playlist_name = ? ORDER BY id COLLATE NOCASE ASC", playlist, function(err, rows){ -// var returnThis = []; -// -// for (var i = 0; i < rows.length; i++) { -// -// // var tempName = rows[i].filepath.split('/').slice(-1)[0]; -// var tempName = fe.basename(rows[i].filepath); -// var extension = getFileType(rows[i].filepath); -// var filepath = slash(fe.relative(req.user.musicDir, rows[i].filepath)); // TODO -// -// returnThis.push({name: tempName, file: filepath, filetype: extension }); -// } -// -// res.send(JSON.stringify(returnThis)); -// }); -// -// }); -// -// -// mstream.get('/deleteplaylist', function(req, res){ -// var playlistname = req.query.playlistname; -// -// // Handle a soft delete -// if(req.query.hide && parseInt(req.query.hide) === 1 ){ -// db.run("UPDATE mstream_playlists SET hide = 1 WHERE playlist_name = ?;", playlistname, function(){ -// res.send('DONE'); -// -// }); -// }else{ // Permentaly delete -// -// // Delete playlist from DB -// db.run("DELETE FROM mstream_playlists WHERE playlist_name = ?;", playlistname, function(){ -// res.send('DONE'); -// -// }); -// } -// }); - - // Download a zip file of music mstream.post('/download', function (req, res){ var archive = archiver('zip'); - archive.on('error', function(err) { console.log(err.message); - res.status(500).send('{error: err.message}'); + res.status(500).send('{error: '+err.message+'}'); }); archive.on('end', function() { // TODO: add logging - console.log('Archive wrote %d bytes', archive.pointer()); }); //set the archive name @@ -587,264 +406,55 @@ mstream.post('/download', function (req, res){ // Get the POSTed files var fileArray = JSON.parse(req.body.fileArray); - + //////////////////////////////////////////////////////////// // TODO: Confirm each item in posted data is a real file // - /////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////// for(var i in fileArray) { var fileString = fileArray[i]; archive.file(fe.normalize( fileString), { name: fe.basename(fileString) }); } - - // TODO: Recursivly download a posted directory // - ////////////////////////////////////////////////// - // SEE: https://github.com/archiverjs/node-archiver/tree/master/examples - // var directory = req.body.directory; - archive.finalize(); }); -// Old way -//const mstreamDB = require('./modules/database-'+program.databaseplugin+'.js'); -// mstreamDB.setup(mstream, program.users, db); // TODO: ROOTDIR -// New Way +// ============================================================================ +// +// // Old way +// //const mstreamDB = require('./modules/database-'+program.databaseplugin+'.js'); +// // mstreamDB.setup(mstream, program.users, db); // TODO: ROOTDIR +// +// // New Way +// // TODO: We need to pull this from the program var var publicDBType = 'sqlite3'; // Can be sqlite3/mysql/LokiJS +var dbSettings = program.database_plugin; const mstreamDB = require('./modules/db-management/database-master.js'); -mstreamDB.setup(mstream, program.users, publicDBType); - - -// mstream.post('/db/search', function(req, res){ -// var searchTerm = "%" + req.body.search + "%" ; -// -// var returnThis = {"albums":[], "artists":[]}; -// -// // TODO: Combine SQL calls into one -// db.serialize(function() { -// -// var sqlAlbum = "SELECT DISTINCT album FROM items WHERE items.album LIKE ? ORDER BY album COLLATE NOCASE ASC;"; -// db.all(sqlAlbum, searchTerm, function(err, rows) { -// if(err){ -// res.status(500).json({ error: 'DB Error' }); -// return; -// } -// -// for (var i = 0; i < rows.length; i++) { -// if(rows[i].album){ -// // rows.splice(i, 1); -// returnThis.albums.push(rows[i].album); -// } -// } -// }); -// -// -// var sqlAlbum = "SELECT DISTINCT artist FROM items WHERE items.artist LIKE ? ORDER BY artist COLLATE NOCASE ASC;"; -// db.all(sqlAlbum, searchTerm, function(err, rows) { -// if(err){ -// res.status(500).json({ error: 'DB Error' }); -// return; -// } -// -// for (var i = 0; i < rows.length; i++) { -// if(rows[i].artist){ -// // rows.splice(i, 1); -// returnThis.artists.push(rows[i].artist); -// } -// } -// -// res.send(JSON.stringify(returnThis)); -// -// }); -// }); -// }); - - - -// mstream.get('/db/artists', function (req, res) { -// var sql = "SELECT DISTINCT artist FROM items ORDER BY artist COLLATE NOCASE ASC;"; -// -// var artists = {"artists":[]}; -// -// db.all(sql, function(err, rows) { -// if(err){ -// res.status(500).json({ error: 'DB Error' }); -// return; -// } -// -// var returnArray = []; -// for (var i = 0; i < rows.length; i++) { -// if(rows[i].artist){ -// // rows.splice(i, 1); -// artists.artists.push(rows[i].artist); -// } -// } -// -// res.send(JSON.stringify(artists)); -// }); -// }); - - - -// mstream.post('/db/artists-albums', function (req, res) { -// var sql = "SELECT DISTINCT album FROM items WHERE artist = ? ORDER BY album COLLATE NOCASE ASC;"; -// -// var searchTerm = req.body.artist ; -// -// var albums = {"albums":[]}; -// -// // TODO: Make a list of all songs without null albums and add them to the response -// -// -// db.all(sql, searchTerm, function(err, rows) { -// if(err){ -// res.status(500).json({ error: 'DB Error' }); -// return; -// } -// -// -// var returnArray = []; -// for (var i = 0; i < rows.length; i++) { -// if(rows[i].album){ -// // rows.splice(i, 1); -// albums.albums.push(rows[i].album); -// } -// } -// -// res.send(JSON.stringify(albums)); -// }); -// }); - - - -// mstream.get('/db/albums', function (req, res) { -// var sql = "SELECT DISTINCT album FROM items ORDER BY album COLLATE NOCASE ASC;"; -// -// var albums = {"albums":[]}; -// -// -// db.all(sql, function(err, rows) { -// if(err){ -// res.status(500).json({ error: 'DB Error' }); -// return; -// } -// -// -// var returnArray = []; -// for (var i = 0; i < rows.length; i++) { -// if(rows[i].album){ -// albums.albums.push(rows[i].album); -// -// } -// } -// -// console.log(JSON.stringify(albums)); -// res.send(JSON.stringify(albums)); -// }); -// }); - - - -// mstream.post('/db/album-songs', function (req, res) { -// var sql = "SELECT title, artist, album, format, year, cast(path as TEXT), track FROM items WHERE album = ? ORDER BY track ASC;"; -// var searchTerm = req.body.album ; -// -// -// -// db.all(sql, searchTerm, function(err, rows) { -// if(err){ -// res.status(500).json({ error: 'DB Error' }); -// return; -// } -// -// // Format data for API -// // rows = setLocalFileLocation(rows); -// for(var i in rows ){ -// var path = String(rows[i]['cast(path as TEXT)']); -// -// rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase -// rows[i].file_location = slash(fe.relative(req.user.musicDir, path)); // Get the local file location -// rows[i].filename = fe.basename( path ); // Ge the filname -// } -// -// -// res.send(JSON.stringify(rows)); -// }); -// }); -// // // TODO -// // function setLocalFileLocation(rows){ -// // -// // for(var i in rows ){ -// // var path = String(rows[i]['cast(path as TEXT)']); -// // -// // rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase -// // rows[i].file_location = slash(fe.relative(rootDir, path)); // Get the local file location -// // rows[i].filename = fe.basename( path ); // Ge the filname -// // } -// // -// // return rows; -// // } - +// mstreamDB.setup(mstream, program.users, publicDBType, dbSettings); +mstreamDB.setup(mstream, program); +// ============================================================================ // TODO: Add individual song -// mstream.get('/db/add-songs', function(req, res){ -// // deseralize json array -// // Add all files -// }); - - -// Download the database -mstream.get('/db/download-db', function(req, res){ - var file = program.database; - - res.download(file); // Set disposition and send it. +mstream.get('/db/add-songs', function(req, res){ + res.send('Coming Soon!'); }); - -// Get hash of database -mstream.get( '/db/hash', function(req, res){ - var hash = crypto.createHash('sha256'); - var fileStream = fs.createReadStream(program.database); - - hash.setEncoding('hex'); - fileStream.pipe(hash, { end: false }); - - - fileStream.on('end', function () { - hash.end(); - - var returnThis = { - hash:String(hash.read()) - }; - - res.send(JSON.stringify(returnThis)); - - }); -}); - - // TODO: Get Album Art calls mstream.post( '/get-album-art', function(req, res){ // Get filepath from post - // Check if album art is in DB // Return If So - // Pull album art from file stream - // ??? Lookup album art via 3rd party ??? res.send('Coming Soon!'); - -} - -const server = mstream.listen(port, function () { - // var host = server.address().address; - // var port = server.address().port; - // console.log('Example app listening at http://%s:%s', host, port); }); + + +// Start the server! +const server = mstream.listen(program.port, function () {}); diff --git a/package.json b/package.json index e7956bf..e15fcf7 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,6 @@ "body-parser": "^1.15.1", "commander": "^2.9.0", "express": "^4.13.4", - "graceful-fs": "^4.1.4", "jsonwebtoken": "^7.1.9", "musicmetadata": "^2.0.3", "my-local-ip": "^1.0.0", diff --git a/public/js/mstream.js b/public/js/mstream.js index 25deb3c..ecd72f4 100755 --- a/public/js/mstream.js +++ b/public/js/mstream.js @@ -37,6 +37,7 @@ $(document).ready(function(){ // Add the token the URL calls accessKey = token; + virtualDirectory = parsedResponse.vPath; loadFileExplorer(); // Remove the overlay @@ -57,6 +58,7 @@ $(document).ready(function(){ var accessKey = ''; + var virtualDirectory = ''; $.ajaxPrefilter(function( options ) { options.beforeSend = function (xhr) { xhr.setRequestHeader('x-access-token', accessKey); @@ -81,6 +83,10 @@ $(document).ready(function(){ request.done(function( msg ) { // Remove login screen + // TODO: set virtualDirectory + var decoded = $.parseJSON(msg); + virtualDirectory = decoded.vPath; + }); request.fail(function( jqXHR, textStatus ) { @@ -200,6 +206,12 @@ $(document).ready(function(){ function addFile2(that){ var filename = $(that).attr("id"); var file_location = $(that).data("file_location"); + if(virtualDirectory){ + file_location = virtualDirectory + '/' + file_location; + } + if(accessKey){ + file_location += '?token=' + accessKey; + } var filetype = $(that).data("filetype"); var title = $(that).find('span.title').html(); diff --git a/untitled folder/JSONsqlite.json b/untitled folder/JSONsqlite.json new file mode 100644 index 0000000..896d9a9 --- /dev/null +++ b/untitled folder/JSONsqlite.json @@ -0,0 +1,31 @@ +{ + "port":3031, + "users":[ + { + "username":"paul", + "password":"glassjar98", + "email":"paul@wall.com", + "musicDir":"/path/to/music", + "guest":true, + "guestUsername":"guest name", + "guestPassword":"abcd", + "privateDB":"BEETS", + "privateDBOptions":{ + "importDB":"path/to/sqlite3.db", + "beetspath":"/path/to/beets/music/dir" + }, + "bannedGuestFunctions":"todo. make this optional" + }, + { + "username":"paul2", + "password":"glassjar98", + "email":"paul@wall2.com", + "musicDir":"/path/to/music2", + } + ], + "userinterface":"public", + "database_plugin":{ + "type":"sqlite", + "dbPath":"/path/to/db", + } +} diff --git a/untitled folder/exampleNewJSON.json b/untitled folder/exampleNewJSON.json index 865129d..a7a7537 100644 --- a/untitled folder/exampleNewJSON.json +++ b/untitled folder/exampleNewJSON.json @@ -1,6 +1,5 @@ { "port":3031, - "login":true, "users":[ { "username":"paul", @@ -8,10 +7,14 @@ "email":"paul@wall.com", "musicDir":"/path/to/music", "guest":true, + "guestUsername":"guest name", "guestPassword":"abcd", - "bannedGuestFunctions":"todo. make this optional", - "importDB":"path/to/sqlite3.db", - "beetspath":"/path/to/beets/music/dir" + "privateDB":"BEETS", + "privateDBOptions":{ + "importDB":"path/to/sqlite3.db", + "beetspath":"/path/to/beets/music/dir" + }, + "bannedGuestFunctions":"todo. make this optional" }, { "username":"paul2", @@ -19,22 +22,22 @@ "email":"paul@wall2.com", "musicDir":"/path/to/music2", "guest":"false", - "importDB":"path/to/sqlite3.db" } ], "userinterface":"public", - - "database_public-DEFAULT EXAMPLE SSQLITE3":{ + "database_plugin":{ "type":"sqlite", - "db":"/path/to/db", + "dbPath":"/path/to/db", }, - "database_public-MYSQL":{ + "database_plugin-MYSQL":{ "type":"mysql", "username":"lol", "password":"lol", "db-name":"lolDB" }, - "database-public-LOKI":{ + "database_plugin-LOKI":{ "comming":"soon", - } + }, + "secret":"SECRET GOES HERE. OR A PATH TO THE SECRET", + "tunnel":true, } From 6824ea7f62d8a159e286803a7ac6c66315c44588 Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Wed, 30 Nov 2016 16:46:19 -0500 Subject: [PATCH 05/11] All core functions work. Some secondary functions are still broken or missing --- README.md | 12 +- .../db-management/database-default-manager.js | 131 ++++++++++-------- modules/db-management/database-master.js | 16 +-- modules/db-read/database-public-sqlite.js | 11 +- modules/db-write/database-default-sqlite.js | 42 +++--- mstream.js | 2 +- 6 files changed, 119 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 2c813d0..09fc2de 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ Check it out: http://darncoyotes.mstream.io/ ### Main Features * Supports FLAC streaming -* Built in SQLite DB. No need to setup MySQL +* DB Plugin System. Use SQLite, MySQL, LokiJS or roll your own custom DB system * Works on Mac, Linux and Windows * [Integrates easily with Beets DB](https://github.com/beetbox/beets) -* User system with one admin and one guest account +* Allows multiple users ## Installation @@ -145,6 +145,8 @@ Please note that not all routers will allow this. Some routers may close this p ## TODO -- GET request to jump to playlist or directory -- Look into taglib for id3 info -- SSL support +- Album Art +- Reset Password Functions +- Ability to store hashed passwords +- Scripts that help construct configs +- MySQL DB plugin diff --git a/modules/db-management/database-default-manager.js b/modules/db-management/database-default-manager.js index 7c80302..af90e10 100644 --- a/modules/db-management/database-default-manager.js +++ b/modules/db-management/database-default-manager.js @@ -1,36 +1,36 @@ #!/usr/bin/env node +"use strict"; + // This is designed to run as it's own process // It takes in a json array -// { +// { // "username":"lol", -// "userDir":"/path/to/dir", -// "dbType":"sqlite", -// "dbSettings":{} +// "userDir":"/Users/psori/Desktop/Blockhead", +// "dbSettings":{ +// "type":"sqlite", +// "dbPath":"/Users/psori/Desktop/LATESTGREATEST.DB" +// } // } + const metadata = require('musicmetadata'); -const fs = require('fs'); // File System +const fs = require('fs'); const fe = require('path'); const crypto = require('crypto'); -const fe = require('path'); try{ - if(fe.extname(process.argv[process.argv.length-1]) == '.json' && fs.statSync(process.argv[process.argv.length-1]).isFile()){ - var loadJson = JSON.parse(fs.readFileSync(args[args.length-1], 'utf8')); - }else{ - console.log('Bad input'); - process.exit(); - } + var loadJson = JSON.parse(process.argv[process.argv.length-1], 'utf8'); + }catch(error){ console.log('JSON file does not appear to exist'); process.exit(); } + // TODO: Check JSON for nencessary info -// TODO: Call Function // 2.0 @@ -40,63 +40,61 @@ try{ // Send these arrays to functions in database-default-X.js -var arrayOfSongs; // Holds songs for DB to process // TODO: Move out of global scope +var arrayOfSongs = []; // Holds songs for DB to process // TODO: Move out of global scope var arrayOfScannedFiles = []; // Holds files for from recursive scan -var parseFilesGenerator; // This Generator is used in two places. Should it be seperated? -// var scanDirLock = false; //TODO: Pull in correct module - -const dbRead = require('./modules/db-write/database-default-'+loadJson.dbType+'.js'); -if(loadJson.dbType == 'sqlite'){ - dbRead.setup(loadJson.dbSettings.path); // TODO: Pass this in +console.log(loadJson.dbSettings.type); +const dbRead = require('../db-write/database-default-'+loadJson.dbSettings.type+'.js'); +if(loadJson.dbSettings.type == 'sqlite'){ + dbRead.setup(loadJson.dbSettings.dbPath); // TODO: Pass this in } -function rescanAllDirectoriesWrapper(){ - // if(scanDirLock === true){ - // // TODO: If scanlock == true, aleart user to try again once scanning is done - // // TODO: Enable Button - // return; - // } - // scanDirLock = true; - // TODO: Disable Button - - parseFilesGenerator = rescanAllDirectories(dir); - parseFilesGenerator.next(); -} +// New way to start it +const parseFilesGenerator = rescanAllDirectories(loadJson.userDir); +parseFilesGenerator.next(); +// Old way to start it +// rescanAllDirectoriesWrapper(loadJson.userDir); +// function rescanAllDirectoriesWrapper(dir){ +// parseFilesGenerator = rescanAllDirectories(dir); +// parseFilesGenerator.next(); +// } function *rescanAllDirectories(directoryToScan){ - // Scan the directory for new, modified, and deleted files var filesToProcess = yield rescanDirectory(directoryToScan); // Process all new files - while(filesToProcess.newFiles.length > 0) { - // TODO: Break into chuncks and send to dbRead - yield parseFile(filesToProcess.newFiles.pop()); + if(filesToProcess.newFiles.length != 0){ + while(filesToProcess.newFiles.length > 0) { + yield parseFile(filesToProcess.newFiles.pop()); + } + // Finish inserting all new entries + yield insertEntries(50, true); } + // process all updated files while(filesToProcess.updatedFiles.length > 0) { - // TODO: Break into chuncks and send to dbRead - yield hashOneUpdatedSong(filesToProcess.updatedFiles.pop()); + // TODO: Handle Editted songs + //yield hashOneUpdatedSong(filesToProcess.updatedFiles.pop()); } - // Re-enable scanning - // scanDirLock = false; + // TODO: Process deleted files + + // Exit + process.exit(0); } - - function rescanDirectory(dir){ // Get all files from DB // TODO: Move This - dbRead.getUserFiles(user, function(rows){ + dbRead.getUserFiles(loadJson, function(rows){ // Scan through files var fileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; @@ -191,7 +189,7 @@ function rescanDirectory(dir){ // We need to prompt users to see if they want to delete files on the server side // We can store a default behaviour - returnArray = { + var returnArray = { "newFiles":arrayOfSongsToProcess, "updatedFiles":arrayOfUpdatedSongsToProcess, "deletedFiles":deletedFiles @@ -219,14 +217,17 @@ function parseFile(thisSong){ hash.setEncoding('hex'); var readableStream = fs.createReadStream(thisSong); - var parser = metadata(readableStream, function (err, songInfo) { + var parser = metadata(readableStream, function (err, thisMetadata) { if(err){ // TODO: Do something } - songInfo = thisSong; + console.log(songInfo); + console.log(filestat); + + songInfo = thisMetadata; songInfo.filesize = filestat.size; songInfo.created = filestat.birthtime.getTime(); - songInfo.modified = ilestat.mtime.getTime(); + songInfo.modified = filestat.mtime.getTime(); songInfo.filePath = thisSong; songInfo.format = getFileType(thisSong); @@ -237,16 +238,21 @@ function parseFile(thisSong){ readableStream.close(); songInfo.hash = String(hash.read()); + + console.log('XXXXXXXXXXXXXXXxx'); + console.log(songInfo); + arrayOfSongs.push(songInfo); // if there are more than 100 entries, or if it's the last song if(arrayOfSongs.length > 99){ - //TODO: Need to move this function - insertEntries(); + // Insert entries into DB + insertEntries(99, false); + }else{ + // For the generator + parseFilesGenerator.next(); } - // For the generator - parseFilesGenerator.next(); }); @@ -283,7 +289,24 @@ function recursiveScan(dir, fileTypesArray){ } } -// TODO: -function insertEntries(){ - dbRead.sendUserFiles(); + +function insertEntries(numberToInsert = 99, loopToEnd = false){ + var insertThese = []; + + while(insertThese.length != numberToInsert ){ + if(arrayOfSongs.length == 0){ + break; + } + insertThese.push(arrayOfSongs.pop()); + } + + dbRead.insertEntries(insertThese, loadJson.username, function(){ + // Recursivly run this function until all songs have been added + if(loopToEnd && arrayOfSongs.length != 0){ + insertEntries(numberToInsert, true); + }else{ + // For the generator + parseFilesGenerator.next(); + } + }); } diff --git a/modules/db-management/database-master.js b/modules/db-management/database-master.js index a2ba3f6..ed36d7f 100644 --- a/modules/db-management/database-master.js +++ b/modules/db-management/database-master.js @@ -67,15 +67,15 @@ exports.setup = function(mstream, program){ dbSettings:dbSettings } - const forkedScan = child.fork(__dirname + '/modules/db-management/database-default-manager.js', [JSON.stringify(jsonLoad)]); + const forkedScan = child.fork(__dirname + '/database-default-manager.js', [JSON.stringify(jsonLoad)]); - forkedScan.stdout.on('data', (data) => { - console.log(`stdout: ${data}`); - }); - - forkedScan.stderr.on('data', (data) => { - console.log(`stderr: ${data}`); - }); + // forkedScan.stdout.on('data', (data) => { + // console.log(`stdout: ${data}`); + // }); + // + // forkedScan.stderr.on('data', (data) => { + // console.log(`stderr: ${data}`); + // }); forkedScan.on('close', (code) => { userDBStatus[user.username] = false; diff --git a/modules/db-read/database-public-sqlite.js b/modules/db-read/database-public-sqlite.js index 5eaa975..b1276f8 100644 --- a/modules/db-read/database-public-sqlite.js +++ b/modules/db-read/database-public-sqlite.js @@ -1,4 +1,7 @@ const sqlite3 = require('sqlite3').verbose(); +const slash = require('slash'); +const fe = require('path'); + // function that takes in a json array of songs and saves them to the sqlite db @@ -20,7 +23,7 @@ exports.setup = function(mstream, dbSettings){ const db = new sqlite3.Database(dbSettings.dbPath); // Setup DB - db.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER, user VARCHAR, filesize INTEGER, file_created_date INTEGER, file_modified_date INTEGER);", function() { + db.run("CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path TEXT NOT NULL UNIQUE, format VARCHAR, track INTEGER, disk INTEGER, user VARCHAR, filesize INTEGER, file_created_date INTEGER, file_modified_date INTEGER);", function() { }); // Create a playlist table db.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, user VARCHAR, created datetime default current_timestamp);", function() { @@ -149,7 +152,7 @@ exports.setup = function(mstream, dbSettings){ }); - var sqlArtist = "SELECT DISTINCT artist FROM items WHERE items.artist LIKE ? ORDER BY artist COLLATE NOCASE ASC;"; + var sqlArtist = "SELECT DISTINCT artist FROM items WHERE items.artist LIKE ? AND user = ? ORDER BY artist COLLATE NOCASE ASC;"; db.all(sqlArtist, [searchTerm, req.user.username], function(err, rows) { if(err){ res.status(500).json({ error: 'DB Error' }); @@ -218,8 +221,8 @@ exports.setup = function(mstream, dbSettings){ mstream.get('/db/albums', function (req, res) { var albums = {"albums":[]}; - var sql = "SELECT DISTINCT album FROM items ORDER BY album COLLATE NOCASE ASC WHERE user = ?;"; - db.all(sql, [req.user.username], function(err, rows) { + var sql = "SELECT DISTINCT album FROM items WHERE user = ? ORDER BY album COLLATE NOCASE ASC;"; + db.all(sql, req.user.username, function(err, rows) { if(err){ res.status(500).json({ error: 'DB Error' }); return; diff --git a/modules/db-write/database-default-sqlite.js b/modules/db-write/database-default-sqlite.js index 99a1501..eb6b663 100644 --- a/modules/db-write/database-default-sqlite.js +++ b/modules/db-write/database-default-sqlite.js @@ -7,10 +7,12 @@ exports.setup = function(dbPath){ db = new sqlite3.Database(dbPath); } -exports.getUserFiles = function(user, callback){ - db.all("SELECT path, file_modified_date FROM files WHERE user=? ;" [thisUser], function(err, rows){ +exports.getUserFiles = function(thisUser, callback){ + console.log(thisUser.username); + db.all("SELECT path, file_modified_date FROM items WHERE user = ?;", thisUser.username, function(err, rows){ // Format results - var returnThis; + var returnThis = rows; + console.log(rows); // callback function callback(returnThis); @@ -19,14 +21,16 @@ exports.getUserFiles = function(user, callback){ -function insertEntries(){ - var sql2 = "insert into items (title,artist,year,album,path,format, track, disk) values "; +exports.insertEntries = function(arrayOfSongs, username, callback){ + // TODO: Update SQL + var sql2 = "insert into items (title,artist,year,album,path,format, track, disk, user, filesize, file_modified_date, file_created_date) values "; var sqlParser = []; + console.log(arrayOfSongs); + while(arrayOfSongs.length > 0) { var song = arrayOfSongs.pop(); - // console.log(song); var songTitle = null; @@ -51,8 +55,8 @@ function insertEntries(){ songAlbum = song.album; } - - sql2 += "(?, ?, ?, ?, ?, ?, ?, ?), "; + // TODO: Update SQL + sql2 += "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?), "; sqlParser.push(songTitle); sqlParser.push(artistString); sqlParser.push(songYear); @@ -61,6 +65,10 @@ function insertEntries(){ sqlParser.push(song.format); sqlParser.push(song.track.no); sqlParser.push(song.disk.no); + sqlParser.push(username); // TODO: User + sqlParser.push(song.filesize); + sqlParser.push(song.modified); + sqlParser.push(song.created); } @@ -68,20 +76,8 @@ function insertEntries(){ sql2 += ";"; console.log(sql2); - dbCopy.run(sql2, sqlParser); -} - -function prep(){ - dbCopy.serialize(function() { - // These two queries will run sequentially. - dbCopy.run("drop table if exists items;"); - dbCopy.run("CREATE TABLE items ( id INTEGER PRIMARY KEY AUTOINCREMENT, title varchar DEFAULT NULL, artist varchar DEFAULT NULL, year int DEFAULT NULL, album varchar DEFAULT NULL, path text, format varchar, track INTEGER, disk INTEGER);", function() { - // These queries will run in parallel and the second query will probably - // fail because the table might not exist yet. - console.log('TABLES CREATED'); - - parse = parseAllFiles(); - parse.next(); - }); + db.run(sql2, sqlParser, function() { + console.log('ITS DONE'); + callback(); }); } diff --git a/mstream.js b/mstream.js index fb9ef08..f71b65b 100755 --- a/mstream.js +++ b/mstream.js @@ -277,7 +277,7 @@ if(program.users){ next(); }); - mstream.use( '/' , express.static( program.users[i].musicDir )); + mstream.use( '/' , express.static( process.cwd() )); } From cefefffaa40ea9b0247cf4b28d10c4f235acfd3f Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Fri, 2 Dec 2016 00:09:57 -0500 Subject: [PATCH 06/11] Fixed broken API endpoint --- modules/db-read/database-public-sqlite.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/db-read/database-public-sqlite.js b/modules/db-read/database-public-sqlite.js index b1276f8..2fdda92 100644 --- a/modules/db-read/database-public-sqlite.js +++ b/modules/db-read/database-public-sqlite.js @@ -93,7 +93,7 @@ exports.setup = function(mstream, dbSettings){ mstream.get('/loadplaylist', function (req, res){ var playlist = req.query.playlistname; - db.all("SELECT * FROM mstream_playlists WHERE playlist_name = ? ORDER BY id COLLATE NOCASE ASC", [playlist, req.user.username], function(err, rows){ + db.all("SELECT * FROM mstream_playlists WHERE playlist_name = ? AND user = ? ORDER BY id COLLATE NOCASE ASC", [playlist, req.user.username], function(err, rows){ var returnThis = []; for (var i = 0; i < rows.length; i++) { From e952357eae4f99af80c810eda409594eca317670 Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Fri, 2 Dec 2016 00:58:29 -0500 Subject: [PATCH 07/11] Code cleanup --- modules/configure-json-file.js | 2 +- .../db-management/database-beets-manager.js | 151 ------------------ modules/db-management/database-master.js | 29 +--- modules/db-read/database-public-sqlite.js | 35 ++++ modules/db-write/database-default-sqlite.js | 5 - 5 files changed, 43 insertions(+), 179 deletions(-) delete mode 100644 modules/db-management/database-beets-manager.js diff --git a/modules/configure-json-file.js b/modules/configure-json-file.js index 0849113..e2544a4 100644 --- a/modules/configure-json-file.js +++ b/modules/configure-json-file.js @@ -20,7 +20,7 @@ exports.setup = function(args){ loadJson.userinterface = "public"; } - + // TODO; Preform a full range of checks // Export JSON return loadJson; diff --git a/modules/db-management/database-beets-manager.js b/modules/db-management/database-beets-manager.js deleted file mode 100644 index a2291f5..0000000 --- a/modules/db-management/database-beets-manager.js +++ /dev/null @@ -1,151 +0,0 @@ -// Special functions for beets DB - -// Download the database -// TODO: Fix these -mstream.get('/db/download-db', function(req, res){ - var file = program.database; - - res.download(file); // Set disposition and send it. -}); - - -// Get hash of database -mstream.get( '/db/hash', function(req, res){ - var hash = crypto.createHash('sha256'); - var fileStream = fs.createReadStream(program.database); - - hash.setEncoding('hex'); - fileStream.pipe(hash, { end: false }); - - - fileStream.on('end', function () { - hash.end(); - - var returnThis = { - hash:String(hash.read()) - }; - - res.send(JSON.stringify(returnThis)); - - }); -}); - - - - - -// // TODO: This thing has to be tested -// -// // TODO: These functions are for interacting withe the beets DB -// // Includes: rescan DB, hash files -// // Once DB has been updated, call functiosn in /db-write/database-beets-[mysql/sqlite/loki].js to pull info into publicDB -// -// const spawn = require('child_process').spawn; -// var scanLock = false; -// var yetAnotherArrayOfSongs = []; -// var totalFileCount = 0; -// -// exports.setup = function(mstream, program, rootDir, db){ -// const scanThisDir = program.beetspath; // TODO: Check that this is a real directory -// -// -// mstream.get('/db/recursive-scan-beets', function(req,res){ -// -// if(scanLock === true){ -// // Return error -// res.status(401).send('{"error":"Scan in progress"}'); -// return; -// } -// -// scanLock = true; -// var cmd = spawn('beet', [ 'import', '-A', '--group-albums' , scanThisDir]); -// -// cmd.stdout.on('data', (data) => { -// console.log(`stdout: ${data}`); -// }); -// -// cmd.stderr.on('data', (data) => { -// console.log(`stderr: ${data}`); -// scanLock = false; -// -// }); -// -// cmd.on('close', (code) => { -// console.log(`child process exited with code ${code}`); -// hashFileBeets(); -// -// // TODO: Remove all empty dirs -// }); -// }); -// -// -// function hashFileBeets(){ -// // var hashCmd = spawn('beet check -a'); -// var hashCmd = spawn('beet', [ 'check', '-a']); -// -// -// hashCmd.stdout.on('data', (data) => { -// console.log(`stdout: ${data}`); -// }); -// -// hashCmd.stderr.on('data', (data) => { -// console.log(`stderr: ${data}`); -// scanLock = false; -// -// }); -// -// hashCmd.on('close', (code) => { -// console.log(`child process exited with code ${code}`); -// scanLock = false; -// -// }); -// } -// -// // TODO: Function that will remove all empty folders -// function removeEmptyFolders(){ -// var hashCmd = spawn('beet', [ 'check', '-a']); -// // 'find ~ -type d -empty -delete' -// } -// -// -// -// mstream.get('/db/status-beets', function(req, res){ -// var returnObject = {}; -// -// returnObject.locked = scanLock; -// -// -// if(scanLock){ -// -// // Currently we don't support filecount stats when using beets DB -// // Dummy data -// returnObject.totalFileCount = 0; -// returnObject.filesLeft = 0; -// -// -// res.json(returnObject); -// -// }else{ -// var sql = 'SELECT Count(*) FROM items'; -// -// db.get(sql, function(err, row){ -// if(err){ -// console.log(err.message); -// -// res.status(500).json({ error: err.message }); -// return; -// } -// -// -// var fileCountDB = row['Count(*)']; // TODO: Is this correct??? -// -// returnObject.totalFileCount = fileCountDB; -// res.json(returnObject); -// -// }); -// } -// -// }); -// -// -// } diff --git a/modules/db-management/database-master.js b/modules/db-management/database-master.js index ed36d7f..95ebfde 100644 --- a/modules/db-management/database-master.js +++ b/modules/db-management/database-master.js @@ -50,12 +50,17 @@ exports.setup = function(mstream, program){ /////////////////////////// // TODO: We could use some kind of manager to make sure we don't spawn to many child processes - // For know we spawn indiscriminately and let the CPU sort it out + // For now we spawn indiscriminately and let the CPU sort it out /////////////////////////// // TODO: Fill this out function forkBeets(user, publicDBType, dbSettings){ + // Pull beets commands from config + // Run commands + // beet import -A --group-albums /path/to/music + // beet check -a + // find ~ -type d -empty -delete } function forkDefault(user, dbSettings){ @@ -81,8 +86,6 @@ exports.setup = function(mstream, program){ userDBStatus[user.username] = false; console.log(`child process exited with code ${code}`); }); - - // TODO: Need to make an on error } @@ -98,25 +101,7 @@ exports.setup = function(mstream, program){ // TODO: Handle user status mstream.get('/db/status-mstream', function(req, res){ + res.send('Coming Soon!'); }); - - - - - - - - - -// TODO: Load any plugins necessary for habdling indivudal user dbs - // Then construct routing between api calls and userDB management functions - - - // TODO: Handle Specialized DB Functions - // mstream.get('/db/download-db', function(req, res){ - // }); - // mstream.get( '/db/hash', function(req, res){ - // }); - } diff --git a/modules/db-read/database-public-sqlite.js b/modules/db-read/database-public-sqlite.js index 2fdda92..2ba09d4 100644 --- a/modules/db-read/database-public-sqlite.js +++ b/modules/db-read/database-public-sqlite.js @@ -1,6 +1,7 @@ const sqlite3 = require('sqlite3').verbose(); const slash = require('slash'); const fe = require('path'); +const crypto = require('crypto'); @@ -266,4 +267,38 @@ exports.setup = function(mstream, dbSettings){ }); }); + mstream.get('/db/download-db', function(req, res){ + // Check user for beets db + if(!req.user.privateDB || req.user.privateDB != 'BEETS'){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + // Download File + res.download(req.user.privateDBOptions.importDB); + }); + + + // Get hash of database + // TODO: Change the name of this endpoint + mstream.get( '/db/hash', function(req, res){ + // Check if user is using beets + if(!req.user.privateDB || req.user.privateDB != 'BEETS'){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + var hash = crypto.createHash('sha256'); + hash.setEncoding('hex'); + + var fileStream = fs.createReadStream(req.user.privateDBOptions.importDB); + fileStream.on('end', function () { + hash.end(); + res.json( {hash:String(hash.read())} ); + }); + + fileStream.pipe(hash, { end: false }); + }); + + } diff --git a/modules/db-write/database-default-sqlite.js b/modules/db-write/database-default-sqlite.js index eb6b663..bf59c39 100644 --- a/modules/db-write/database-default-sqlite.js +++ b/modules/db-write/database-default-sqlite.js @@ -22,17 +22,12 @@ exports.getUserFiles = function(thisUser, callback){ exports.insertEntries = function(arrayOfSongs, username, callback){ - // TODO: Update SQL var sql2 = "insert into items (title,artist,year,album,path,format, track, disk, user, filesize, file_modified_date, file_created_date) values "; var sqlParser = []; - console.log(arrayOfSongs); - while(arrayOfSongs.length > 0) { var song = arrayOfSongs.pop(); - - var songTitle = null; var songYear = null; var songAlbum = null; From cb3293359c599d827d0faee8819f2cbcf2f4c49c Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Sun, 11 Dec 2016 04:04:14 -0500 Subject: [PATCH 08/11] Lots of new stuff --- README.md | 52 ++-- modules/auto-port-forwarding.js | 62 +++-- modules/configure-commander.js | 79 ++++-- modules/configure-json-file.js | 70 +++++- .../db-management/database-beets-manager.js | 71 ++++++ .../db-management/database-default-manager.js | 12 +- modules/db-management/database-master.js | 36 +++ modules/db-read/database-public-pouch.js | 0 modules/db-read/database-public-sqlite.js | 44 +--- modules/db-write/database-beets-mysql.js | 1 - modules/db-write/database-beets-sqlite.js | 1 - modules/db-write/database-default-pouch.js | 87 +++++++ modules/db-write/database-default-sqlite.js | 19 +- mstream.js | 225 +++++++----------- package.json | 4 +- public/js/mstream.js | 11 +- untitled folder/exampleNewJSON.json | 39 +-- untitled folder/skeleton.json | 4 + 18 files changed, 534 insertions(+), 283 deletions(-) create mode 100644 modules/db-management/database-beets-manager.js create mode 100644 modules/db-read/database-public-pouch.js delete mode 100644 modules/db-write/database-beets-mysql.js delete mode 100644 modules/db-write/database-beets-sqlite.js create mode 100644 modules/db-write/database-default-pouch.js create mode 100644 untitled folder/skeleton.json diff --git a/README.md b/README.md index 09fc2de..4250087 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ ## mStream mStream is an music streaming server written in NodeJS. It's focus is on ease of installation and FLAC streaming. mStream will work right out of the box without any configuration. -### Live Demo +### Demo Check it out: http://darncoyotes.mstream.io/ ### Main Features * Supports FLAC streaming -* DB Plugin System. Use SQLite, MySQL, LokiJS or roll your own custom DB system +* DB Plugin System. Choose the DB that best fits your needs * Works on Mac, Linux and Windows * [Integrates easily with Beets DB](https://github.com/beetbox/beets) * Allows multiple users @@ -55,6 +55,7 @@ sudo npm install -g node-gyp ### Using Docker +##### NOTE: This instructions are outdated and need to be updated Download the Dockerfile, or clone the repository, then run the following commands: @@ -72,23 +73,14 @@ default installation. docker run --rm -v /path/to/my/music:/music local/mstream -l -u username -x password ``` -## Options +## Usage +mStream can be configured by loading a JSON config file ```shell --p, --port -> set port number --l, --login -> enable user login --u, --user -> add user --x, --password -> set Password --G, --guest -> set guest username --X, --guestpassword -> set guest password --d, --database -> set the database file --t, --tunnel -> tunnel --g, --gateway -> set gateway for tunnelling --i, --userinterface -> use an alternative UI. Currently only the value 'jplayer' works +mstream server.json ``` - ## User System The current user system is a simple as it comes. There are two users you can have, main and guest. Guest users do not have any access to API functions that write to the file system. Currently guest users cannot access the save-playlist, recursive-scan, or delete-playlist functions @@ -101,26 +93,37 @@ mstream -l -u [username] -x [password] The user system is simple for a few reasons. First, I wanted to have a user system that doesn't need a database to work. Secondly, mStream is a personal server and most users don't need anything more complex than this. -## Database +## Database Options -mStream currently uses a SQLite database for a music library. You have the option of using a beets DB or having a mStream create it's own DB. +mSTream's datbase will work right out of the box without -#### Beets DB +### Database Plugin System + +mStream 2.0 is written so that you can choose what DB system you use. Currently only sqlite is supported, but in the future their will be more options: + +- SQLite +- MySQL +- PouchDB: A NoSQL alternative + + + + + +### Import DB http://beets.io/ -mStream can use your beets database without any configuration. -```shell -mstream -d path/to/beets.db -``` - -Currently using beets is the recommended way to create a music database. +User's can choose how their files are managed. By default mstream will manage the user's DB. User's also have the option to import their DB from somewhere else +#### beets DB #### use mStream to build your DB Use the /db/recursive-scan API call to kickoff a full scan of your library. Currently this is the only way to add files to the library. Version 2 of mStream will include new functions to update the library more efficiently + + + ## Automatically setup port forwarding #### Please note that this feature is still experimental @@ -150,3 +153,6 @@ Please note that not all routers will allow this. Some routers may close this p - Ability to store hashed passwords - Scripts that help construct configs - MySQL DB plugin +- LokiJS or PuchDB plugin +- Move to LokiJS/PouchDB as default DB +- SSL Support diff --git a/modules/auto-port-forwarding.js b/modules/auto-port-forwarding.js index 85dc394..1fa882b 100644 --- a/modules/auto-port-forwarding.js +++ b/modules/auto-port-forwarding.js @@ -2,10 +2,15 @@ const natupnp = require('nat-upnp'); const getIP = require('external-ip')(); const natpmp = require('nat-pmp'); +var gateway; -exports.tunnel_uPNP = function(port){ +function set_gateway (gateIP){ + gateway = gateIP; +} + + +function tunnel_uPNP (port){ try{ - console.log('Preparing to tunnel via upnp protocol'); var client = natupnp.createClient(); @@ -29,22 +34,14 @@ exports.tunnel_uPNP = function(port){ } } -exports.tunnel_NAT_PMP = function tunnel_NAT_PMP(port){ +function tunnel_NAT_PMP(port){ try{ - console.log('Preparing to tunnel via nat-pmp protocol'); - - // Use the user supplied Gateway IP or try to find it manually - if(program.gateway){ - var gateway = program.gateway; - }else{ - var netroute = require('netroute'); - var gateway = netroute.getGateway(); + if(!gateway){ + gateway = require('netroute').getGateway(); } - console.log('Attempting to tunnel via gateway: ' + gateway); - var client = new natpmp.Client(gateway); client.portMapping({ public: port, private: port }, function (err, info) { if (err) { @@ -52,9 +49,6 @@ exports.tunnel_NAT_PMP = function tunnel_NAT_PMP(port){ } client.close(); }); - - - } catch (e) { console.log('WARNING: mStream nat-pmp tunnel functionality has failed. Your network may not allow functionality'); @@ -64,7 +58,7 @@ exports.tunnel_NAT_PMP = function tunnel_NAT_PMP(port){ -exports.logUrl = function(port){ +function logUrl (port){ getIP(function (err, ip) { if (err) { // every service in the list has failed @@ -73,3 +67,37 @@ exports.logUrl = function(port){ console.log('Access mStream on the internet: http://' + ip + ':' + port); }); } + + +exports.setup = function(args, port){ + if(args.gateway){ + tunnel.set_gateway(args.gateway); + } + + console.log('Preparing to tunnel via nat-pmp protocol'); + + // TODO: Clean this up, this it so lazy... + if(args.protocol && args.protocol === 'upnp'){ + // Run it on an interval ? + if(args.refreshInterval){ + setInterval( function() { + tunnel_uPNP(port); + }, args.refreshInterval); + }else{ + tunnel_uPNP(port); + } + }else{ + // Run it on an interval ? + if(args.refreshInterval){ + setInterval( function() { + tunnel_NAT_PMP(port); + }, argsrefreshInterval); + }else{ + tunnel_NAT_PMP(port); + } + } + + + + logUrl(port); +} diff --git a/modules/configure-commander.js b/modules/configure-commander.js index 5074c9d..8fe60cd 100644 --- a/modules/configure-commander.js +++ b/modules/configure-commander.js @@ -1,29 +1,78 @@ exports.setup = function(args){ - - // TODO: This exists to have a simple setup from the command line - // Removed beets support from cli launch - - // Setup Command Line Interface - var program = require('commander'); + const program = require('commander'); program - .version('1.30.0') + .version('1.21.0') .option('-p, --port ', 'Select Port', /^\d+$/i, 3000) .option('-t, --tunnel', 'Use nat-pmp to configure port fowarding') - .option('-g, --gateip ', 'Manually set gateway IP for the tunnel option') + .option('-g, --gateway ', 'Manually set gateway IP for the tunnel option') + .option('-r, --refresh ', 'Refresh rate', /^\d+$/i) + .option('-o, --protocol ', 'Refresh rate', /^\d+$/i) .option('-u, --user ', 'Set Username') .option('-x, --password ', 'Set Password') - // .option('-e, --email ', 'Set User Email (optional)') + .option('-e, --email ', 'Set User Email (optional)') .option('-G, --guest ', 'Set Guest Username') .option('-X, --guestpassword ', 'Set Guest Password') - // .option('-k, --key ', 'Add SSL Key') - // .option('-c, --cert ', 'Add SSL Certificate') .option('-d, --database ', 'Specify Database Filepath', 'mstreamdb.lite') - // .option('-b, --beetspath ', 'Specify Folder where Beets DB should import music from. This also overides the normal DB functions with functions that integrate with beets DB') - // .option('-b, --databaseplugin ', '', /^(default|beets)$/i, 'default') .option('-i, --userinterface ', 'Specify folder name that will be served as the UI', 'public') - .option('-f, --filepath ', 'Set the path of your music directory', process.cwd()) .option('-s, --secret ', 'Set the login secret key') .parse(args); - return program; + + let program3 = { + port:program.port, + userinterface:program.userinterface, + } + + if(program.secret){ + program3.secret = program.sectet; + } + if(program.salt){ + program3.salt = program.salt; + } + + // User account + if(program.user && program.password){ + program3.users = {}; + program3.users[program.user] = { + password:program.password, + musicDir:process.cwd() + }; + + if(program.email){ + program3.users[program.user].email = program.email; + } + + // Guest account + if(program.guestname && program.guestpassword){ + program3.users[program.guestname] = { + password:program.guestpassword, + guestTo:program.user + }; + } + } + + // db plugins + program3.database_plugin = { + type:"sqlite", + dbPath:program.database + }; + // TODO: Add support for other DBs when ready + + // port forwarding + if(program.tunnel){ + program3.tunnel = {}; + + if(program.refresh){ + program3.tunnel.refreshInterval = program.refresh; + } + if(program.gateway){ + program3.tunnel.gateway = program.gateway; + } + if(program.protocol){ + program3.tunnel.protocol = program.protocol; + } + } + + + return program3; } diff --git a/modules/configure-json-file.js b/modules/configure-json-file.js index e2544a4..e85603d 100644 --- a/modules/configure-json-file.js +++ b/modules/configure-json-file.js @@ -1,27 +1,73 @@ -const fs = require('graceful-fs'); // File System +const fs = require('fs'); // File System +const fe = require('path'); -exports.setup = function(args){ - // Open File +exports.setup = function(args, rootDir){ + let loadJson; try{ - var loadJson = JSON.parse(fs.readFileSync(args[args.length-1], 'utf8')); - }catch(err){ - console.log('Failed to parse JSON file'); - return false; + if(fe.extname(args[args.length-1]) === '.json' && fs.statSync(args[args.length-1]).isFile()){ + loadJson = JSON.parse(fs.readFileSync(args[args.length-1], 'utf8')); + }else{ + return require('./configure-commander.js').setup(args); + } + }catch(error){ + console.log(error); + return {error:"Failed to parse JSON file"}; } + if(!loadJson.port){ + loadJson.port = 5050; + } + if(!isInt(loadJson.port) || loadJson.port < 0 || loadJson.port > 65535){ + return {error:"BAD PORT, WILL ABORT"}; + } + + // TODO: Add comprehensive DB checks if(!loadJson.database_plugin){ - console.log('Please Configure DB'); - return false; + return {error:"Please Configure DB"}; } - if(!loadJson.userinterface){ - loadJson.userinterface = "public"; + + if(loadJson.userinterface){ + if(!fs.statSync( fe.join(rootDir, loadJson.userinterface) ).isDirectory()){ + return {error:"Could not find userinterface"}; + } } - // TODO; Preform a full range of checks + + // Normalize for all OS + // Make sure it's a directory + // Loop through and makeure all user Dirs are real + if(loadJson.users){ + for (let username in loadJson.users) { + // TODO: Check usernames for forbidden chars + + // TODO: Make sure all music directories are unique + // TODO: No subsets/super-sets/duplicates + if(!loadJson.users[username].guestTo && !fs.statSync( loadJson.users[username].musicDir ).isDirectory()){ + return {error:loadJson.users[username].username + " music directory could not be found"}; + } + } + } + + // TODO: Preform a full range of checks + + if(!isInt(loadJson.tunnel.refreshInterval)){ + return {error:"Refresh interval must be an integer"}; + } // Export JSON return loadJson; } + + +function isInt(value) { + if (isNaN(value)) { + return false; + } + var x = parseFloat(value); + return (x | 0) === x; +} + +// TODO: This should sum up all errors before returing to user diff --git a/modules/db-management/database-beets-manager.js b/modules/db-management/database-beets-manager.js new file mode 100644 index 0000000..0f56db5 --- /dev/null +++ b/modules/db-management/database-beets-manager.js @@ -0,0 +1,71 @@ +// TODO: Function that copies a BeetsDB(private) into the SQLite master(public) DB +const sqlite3 = require('sqlite3').verbose(); + +// This is designed to run as it's own process +// It takes in a json array +// { +// "username":"lol", +// "privateDBOptions":{ +// "privateDB":"BEETS", +// "importDB":"path/to/sqlite3.db", +// "beetspath":"/path/to/beets/music/dir", +// "quickSync": true +// }, +// "userDir":"/Users/psori/Desktop/Blockhead", +// "dbSettings":{ +// "type":"sqlite", +// "dbPath":"/Users/psori/Desktop/LATESTGREATEST.DB" +// } +// } + + +try{ + var loadJson = JSON.parse(process.argv[process.argv.length-1], 'utf8'); + +}catch(error){ + console.log('Cannot parse JSON input'); + process.exit(); +} + + +const dbPublic = require('../db-write/database-default-'+loadJson.dbSettings.type+'.js'); +const beetsDB = new sqlite3.Database(dbPath); + +if(loadJson.dbSettings.type == 'sqlite'){ + dbPublic.setup(loadJson.dbSettings.dbPath); // TODO: Pass this in +} + + +run(); + +// TODO: It might be worth combing this into the default-sqlite files since they share some functions + // SELECT * FROM items LEFT JOIN item_attributes ON + // item_attributes.entity_id = items.id + // AND item_attributes.key = 'checksum' + // GROUP BY items.id, item_attributes.key; +function run(){ + let sql = "SELECT * FROM items LEFT JOIN item_attributes ON item_attributes.entity_id = items.id AND item_attributes.key = 'checksum' GROUP BY items.id, item_attributes.key;"; + beetsDB.all(sql, function(err, files){ + files = dbPublic.reformatData(files); + dbPublic.purgeDB(username); + dbPublic.addToDB(files); + + if(loadJson.privateDBOptions.quickSync === false){ + smokeThatHash(); + } + }); +} + + +// TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO: +function smokeThatHash( blazeItEveryDay = false){ + + // Pull all files from DB + // Get hash + // Update DB + + + // if hash is available or blazeItEveryDay=true then hash file + // TODO: Shoudl we add the hash to the beets DB? +} +// TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO: diff --git a/modules/db-management/database-default-manager.js b/modules/db-management/database-default-manager.js index af90e10..0c86361 100644 --- a/modules/db-management/database-default-manager.js +++ b/modules/db-management/database-default-manager.js @@ -23,7 +23,7 @@ try{ var loadJson = JSON.parse(process.argv[process.argv.length-1], 'utf8'); }catch(error){ - console.log('JSON file does not appear to exist'); + console.log('Cannot parse JSON input'); process.exit(); } @@ -44,8 +44,9 @@ var arrayOfSongs = []; // Holds songs for DB to process // TODO: Move out of glo var arrayOfScannedFiles = []; // Holds files for from recursive scan -//TODO: Pull in correct module +// Pull in correct module console.log(loadJson.dbSettings.type); +// TODO: Rename this var const dbRead = require('../db-write/database-default-'+loadJson.dbSettings.type+'.js'); if(loadJson.dbSettings.type == 'sqlite'){ dbRead.setup(loadJson.dbSettings.dbPath); // TODO: Pass this in @@ -55,13 +56,6 @@ if(loadJson.dbSettings.type == 'sqlite'){ // New way to start it const parseFilesGenerator = rescanAllDirectories(loadJson.userDir); parseFilesGenerator.next(); -// Old way to start it -// rescanAllDirectoriesWrapper(loadJson.userDir); -// function rescanAllDirectoriesWrapper(dir){ -// parseFilesGenerator = rescanAllDirectories(dir); -// parseFilesGenerator.next(); -// } - function *rescanAllDirectories(directoryToScan){ diff --git a/modules/db-management/database-master.js b/modules/db-management/database-master.js index 95ebfde..f320b5a 100644 --- a/modules/db-management/database-master.js +++ b/modules/db-management/database-master.js @@ -104,4 +104,40 @@ exports.setup = function(mstream, program){ res.send('Coming Soon!'); }); + + + + mstream.get('/db/download-db', function(req, res){ + // Check user for beets db + if(!req.user.privateDB || req.user.privateDB != 'BEETS'){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + // Download File + res.download(req.user.privateDBOptions.importDB); + }); + + + // Get hash of database + // TODO: Change the name of this endpoint + mstream.get( '/db/hash', function(req, res){ + // Check if user is using beets + if(!req.user.privateDB || req.user.privateDB != 'BEETS'){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + var hash = crypto.createHash('sha256'); + hash.setEncoding('hex'); + + var fileStream = fs.createReadStream(req.user.privateDBOptions.importDB); + fileStream.on('end', function () { + hash.end(); + res.json( {hash:String(hash.read())} ); + }); + + fileStream.pipe(hash, { end: false }); + }); + } diff --git a/modules/db-read/database-public-pouch.js b/modules/db-read/database-public-pouch.js new file mode 100644 index 0000000..e69de29 diff --git a/modules/db-read/database-public-sqlite.js b/modules/db-read/database-public-sqlite.js index 2ba09d4..c5d8fe2 100644 --- a/modules/db-read/database-public-sqlite.js +++ b/modules/db-read/database-public-sqlite.js @@ -57,8 +57,11 @@ exports.setup = function(mstream, dbSettings){ sql2 += "(?, ?, ?), "; sqlParser.push(title); - sqlParser.push( fe.join(req.user.musicDir, song) ); // TODO: User music dir - sqlParser.push( req.user.username ); // TODO: User music dir + // TODO: We need to strip out the vPath + // We need to allow pre-set vPaths in the config file + // Then strip out the vPath here + sqlParser.push( fe.join(req.user.musicDir, song) ); + sqlParser.push( req.user.username ); } @@ -103,7 +106,7 @@ exports.setup = function(mstream, dbSettings){ var tempName = fe.basename(rows[i].filepath); var extension = getFileType(rows[i].filepath); var filepath = slash(fe.relative(req.user.musicDir, rows[i].filepath)); // TODO - + console.log(filepath); returnThis.push({name: tempName, file: filepath, filetype: extension }); } @@ -254,7 +257,6 @@ exports.setup = function(mstream, dbSettings){ } // Format data for API - // rows = setLocalFileLocation(rows); for(var i in rows ){ var path = String(rows[i]['cast(path as TEXT)']); @@ -267,38 +269,4 @@ exports.setup = function(mstream, dbSettings){ }); }); - mstream.get('/db/download-db', function(req, res){ - // Check user for beets db - if(!req.user.privateDB || req.user.privateDB != 'BEETS'){ - res.status(500).json({ error: 'DB Error' }); - return; - } - - // Download File - res.download(req.user.privateDBOptions.importDB); - }); - - - // Get hash of database - // TODO: Change the name of this endpoint - mstream.get( '/db/hash', function(req, res){ - // Check if user is using beets - if(!req.user.privateDB || req.user.privateDB != 'BEETS'){ - res.status(500).json({ error: 'DB Error' }); - return; - } - - var hash = crypto.createHash('sha256'); - hash.setEncoding('hex'); - - var fileStream = fs.createReadStream(req.user.privateDBOptions.importDB); - fileStream.on('end', function () { - hash.end(); - res.json( {hash:String(hash.read())} ); - }); - - fileStream.pipe(hash, { end: false }); - }); - - } diff --git a/modules/db-write/database-beets-mysql.js b/modules/db-write/database-beets-mysql.js deleted file mode 100644 index ce55153..0000000 --- a/modules/db-write/database-beets-mysql.js +++ /dev/null @@ -1 +0,0 @@ -// TODO: Function that copies a BeetsDB(private) into the MySQL master(public) DB diff --git a/modules/db-write/database-beets-sqlite.js b/modules/db-write/database-beets-sqlite.js deleted file mode 100644 index 92e8106..0000000 --- a/modules/db-write/database-beets-sqlite.js +++ /dev/null @@ -1 +0,0 @@ -// TODO: Function that copies a BeetsDB(private) into the SQLite master(public) DB diff --git a/modules/db-write/database-default-pouch.js b/modules/db-write/database-default-pouch.js new file mode 100644 index 0000000..cc4fccb --- /dev/null +++ b/modules/db-write/database-default-pouch.js @@ -0,0 +1,87 @@ +const PouchDB = require('pouchdb'); + + +exports.setup = function(dbName){ + db = new PouchDB(dbName); +} + +exports.getUserFiles = function(thisUser, callback){ + db.all("SELECT path, file_modified_date FROM items WHERE user = ?;", thisUser.username, function(err, rows){ + // Format results + var returnThis = rows; + + // callback function + callback(returnThis); + }); +} + + + +exports.insertEntries = function(arrayOfSongs, username, callback){ + let bulkDocs = []; + + while(arrayOfSongs.length > 0) { + var song = arrayOfSongs.pop(); + + var songTitle = null; + var songYear = null; + var songAlbum = null; + var artistString = null; + + if(song.artist && song.artist.length > 0){ + artistString = ''; + for (var i = 0; i < song.artist.length; i++) { + artistString += song.artist[i] + ', '; + } + artistString = artistString.slice(0, -2); + } + if(song.title && song.title.length > 0){ + songTitle = song.title; + } + if(song.year && song.year.length > 0){ + songYear = song.year; + } + if(song.album && song.album.length > 0){ + songAlbum = song.album; + } + + // TODO: Update SQL + // sqlParser.push(songTitle); + // sqlParser.push(artistString); + // sqlParser.push(songYear); + // sqlParser.push(songAlbum); + // sqlParser.push(song.filePath); + // sqlParser.push(song.format); + // sqlParser.push(song.track.no); + // sqlParser.push(song.disk.no); + // sqlParser.push(username); + // sqlParser.push(song.filesize); + // sqlParser.push(song.modified); + // sqlParser.push(song.created); + +"insert into items (title,artist,year,album,path,format, track, disk, user, filesize, file_modified_date, file_created_date) values "; + + bulkDocs.push( { + _id: song.filePath, + title: songTitle, + artist: artistString, + year: songYear, + album: songAlbum, + path: song.filePath, // FIXME: Redundant data + format:song.format, + track:song.track.no, + disk: song.disk.no, + user:username, + filesize:song.filesize, + file_modified_date:song.modified, + file_created_date:song.created, + }); + + } + + + db.bulkDocs(bulkDocs, function() { + console.log('ITS DONE'); + callback(); + }); +} diff --git a/modules/db-write/database-default-sqlite.js b/modules/db-write/database-default-sqlite.js index bf59c39..661b553 100644 --- a/modules/db-write/database-default-sqlite.js +++ b/modules/db-write/database-default-sqlite.js @@ -8,11 +8,9 @@ exports.setup = function(dbPath){ } exports.getUserFiles = function(thisUser, callback){ - console.log(thisUser.username); db.all("SELECT path, file_modified_date FROM items WHERE user = ?;", thisUser.username, function(err, rows){ // Format results var returnThis = rows; - console.log(rows); // callback function callback(returnThis); @@ -50,7 +48,6 @@ exports.insertEntries = function(arrayOfSongs, username, callback){ songAlbum = song.album; } - // TODO: Update SQL sql2 += "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?), "; sqlParser.push(songTitle); sqlParser.push(artistString); @@ -60,7 +57,7 @@ exports.insertEntries = function(arrayOfSongs, username, callback){ sqlParser.push(song.format); sqlParser.push(song.track.no); sqlParser.push(song.disk.no); - sqlParser.push(username); // TODO: User + sqlParser.push(username); sqlParser.push(song.filesize); sqlParser.push(song.modified); sqlParser.push(song.created); @@ -76,3 +73,17 @@ exports.insertEntries = function(arrayOfSongs, username, callback){ callback(); }); } + + + +// Function that reformats data from beets +exports.reformatData = function(username){ + let sql = "DELETE FROM items WHERE user = ?"; + db.all(sql, username, function(err, rows){ + }); +} + +// Function that removes all files from the given DB +exports.purgeDB = function(){ + +} diff --git a/mstream.js b/mstream.js index f71b65b..7b7a844 100755 --- a/mstream.js +++ b/mstream.js @@ -10,54 +10,29 @@ const archiver = require('archiver'); // Zip Compression const os = require('os'); const crypto = require('crypto'); const slash = require('slash'); +const uuidV4 = require('uuid/v4'); - -// If the user gives a json file then try pulling the config from that -try{ - var startup = 'configure-commander'; - if(fe.extname(process.argv[process.argv.length-1]) == '.json' && fs.statSync(process.argv[process.argv.length-1]).isFile()){ - startup = 'configure-json-file'; - } -}catch(error){ - console.log('JSON file does not appear to exist'); +// Get the server config +const program = require('./modules/configure-json-file.js').setup(process.argv, __dirname); +if(program.error){ + console.log(program.error); process.exit(); } -const program = require('./modules/' + startup + '.js').setup(process.argv); -if(program == false){ - process.exit(); -} - - - - -// Normalize for all OS -// Make sure it's a directory -// Loop through and makeure all user Dirs are real -// TODO: Move all checks to the JSON module -if(program.users){ - for (let i = 0; i < program.users.length; i++) { - //TODO: Check usernames for forbidden chars - - // TODO: Assure all usernames are unique - // TODO: Or update JSON so usernames have to be unique - - // TODO: Assure only one user per filepath - - if(!fs.statSync( program.users[i].musicDir ).isDirectory()){ - console.log(program.users[i].username + " music directory could not be found"); - process.exit(); - } - } -} - - - - // Magic Middleware Things mstream.use(bodyParser.json()); // support json encoded bodies mstream.use(bodyParser.urlencoded({ extended: true })); // support encoded bodies +// Setup WebApp +if(program.userinterface){ + mstream.use( express.static(fe.join(__dirname, program.userinterface) )); + + // Serve the webapp + mstream.get('/', function (req, res) { + res.sendFile( fe.join(program.userinterface, 'mstream.html'), { root: __dirname }); + }); +} + // Print the local network IP console.log('Access mStream locally: http://localhost:' + program.port); @@ -65,30 +40,12 @@ console.log('Access mStream on your local network: http://' + require('my-local- // Handle Port Forwarding -// TODO: Portforwarding could use a feature that re-opens it on a timed interval // TODO: Switch between uPNP and nat-pmp if(program.tunnel){ - const tunnel = require('./modules/auto-port-forwarding.js'); - tunnel.tunnel_uPNP(program.port); - tunnel.logUrl(program.port); + const tunnel = require('./modules/auto-port-forwarding.js').setup(program.tunnel, program.port); } - -// Check that this is a real dir -if(!fs.statSync( fe.join(__dirname, program.userinterface) ).isDirectory()){ - console.log('The userinterface was not found. Closing...'); - process.exit(); -} -mstream.use( express.static(fe.join(__dirname, program.userinterface) )); - -// Serve the webapp -mstream.get('/', function (req, res) { - res.sendFile( fe.join(program.userinterface, 'mstream.html'), { root: __dirname }); -}); - - - // Login functionality if(program.users){ // Use bcrypt for password storage @@ -126,53 +83,52 @@ if(program.users){ // TODO: password change function - if(false == true){ - mstream.post('/change-password-request', function (req, res) { - // Get email address from request - // validate email against user array - // Generate change password token - // Invalidate all other change password tokens - // Email the user the token + mstream.post('/change-password-request', function (req, res) { + // Get email address from request + // validate email against user array + // Generate change password token + // Invalidate all other change password tokens + // Email the user the token - res.sendFile( 'COMING SOON!' ); - }); + res.sendFile( 'COMING SOON!' ); + }); - mstream.post('/change-password', function (req, res){ - // Check token - // Get new password - // Hash password and update user array + mstream.post('/change-password', function (req, res){ + // Check token + // Get new password + // Hash password and update user array - res.sendFile( 'COMING SOON!' ); - }); - } + res.sendFile( 'COMING SOON!' ); + }); // Create the user array - var Users = {}; - // Construct user array - for (let i = 0; i < program.users.length; i++) { - Users[program.users[i].username] = { - musicDir:program.users[i].musicDir, + // var Users = {}; + + var Users = program.users; + for (let username in Users) { + let permissionsMap = {}; + + generateSaltedPassword(username, Users[username]["password"]); + + if(Users[username].guestTo){ + // DO NOTHING! + }else if ( !(Users[username].musicDir in permissionsMap) ){ + // Generate unique vPath if necessary + // Th best way is to store the vPath in the JSON file + if(!Users[username].vPath){ + Users[username].vPath = uuidV4(); + } + + // Add to permissionsMap + permissionsMap[Users[username].musicDir] = Users[username].vPath; + + // Add dir to express + mstream.use( '/' + Users[username].vPath + '/' , express.static( Users[username].musicDir )); + }else{ + Users[username].vPath = permissionsMap[Users[username].musicDir]; } - if(program.users[i].email){ - Users[program.users[i].username].email = program.users[i].email; - } - if(program.users[i].privateDB){ - Users[program.users[i].username].email = program.users[i].privateDB; - } - if(program.users[i].privateDBOptions){ - Users[program.users[i].username].email = program.users[i].privateDBOptions; - } - - generateSaltedPassword(program.users[i].username, program.users[i].password); - - //////////////////////////////// - // TODO: Handle Guest Options // - //////////////////////////////// - - // TODO: We could use a better way of mapping users to vPaths - mstream.use( '/' + program.users[i].username + '/' , express.static( program.users[i].musicDir )); } @@ -226,7 +182,7 @@ if(program.users){ var sendThis = { success: true, message: 'Welcome To mStream', - vPath: user.username, + vPath: user.vPath, token: jwt.sign(user, secret) // Make the token }; @@ -254,25 +210,34 @@ if(program.users){ } // Deny guest access - if(decoded.guest === true && forbiddenFunctions.indexOf(req.path) != -1){ + // TODO: Modify this based on parameters set in json file + if(decoded.guestTo && forbiddenFunctions.indexOf(req.path) != -1){ return res.redirect('/guest-access-denied'); } // Set user request data req.user = decoded; + // + if(decoded.guestTo){ + req.user.username = req.user.guestTo; + // TODO: We should probably set the vPath elsewhere + req.user.vPath = Users[req.user.guestTo].vPath; + req.user.musicDir = Users[req.user.guestTo].musicDir; + + } next(); }); }); // TODO: Middleware that prevents users from accessing another users files - + // TODO: Strip all password info out }else{ // Dummy data mstream.use(function(req, res, next) { req.user = { username:"", - musicDir:program.filepath + musicDir:process.cwd() }; next(); }); @@ -283,18 +248,14 @@ if(program.users){ - - - // Test function // Used to determine the user has a working login token mstream.get('/ping', function(req, res){ // TODO: Guest status - var returnObject = { - vPath: req.user.username, + res.json({ + vPath: req.user.vPath, guest: false - }; - res.send(JSON.stringify(returnObject)); + }); }); @@ -304,20 +265,18 @@ mstream.post('/dirparser', function (req, res) { var directories = []; var filesArray = []; - // Make sure directory exits - // TODO Get music dir from request + // TODO: Make sure path is a sub-path of the user's music dir var path = fe.join(req.user.musicDir, req.body.dir); - // if(path == ""){ - // path = rootDir; - // }else{ - // path = fe.join(rootDir, path); - // } + // Make sure it's a directory + if(!fs.statSync( path).isDirectory()){ + res.status(500).json({ error: 'Not a directory' }); + return; + } // Will only show these files. Prevents people from snooping around // TODO: Move to global vairable - var masterFileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; + const masterFileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; var fileTypesArray; - if(req.body.filetypes){ fileTypesArray = JSON.parse(req.body.filetypes); }else{ @@ -325,19 +284,11 @@ mstream.post('/dirparser', function (req, res) { } - // Make sure it's a directory - if(!fs.statSync( path).isDirectory()){ - // TODO: Write an error output - // 500 Output? - res.send(""); - return; - } - // get directory contents var files = fs.readdirSync( path); // loop through files - for (var i=0; i < files.length; i++) { + for (let i=0; i < files.length; i++) { try{ var stat = fs.statSync(fe.join(path, files[i])); @@ -365,15 +316,14 @@ mstream.post('/dirparser', function (req, res) { } var returnPath = slash( fe.relative(req.user.musicDir, path) ); - if(returnPath.slice(-1) !== '/'){ returnPath += '/'; } - // Combine list of directories and mp3s - var finalArray = { path:returnPath, contents:filesArray.concat(directories)}; - // Send back some JSON - res.send(JSON.stringify(finalArray)); + // Send back combined list of directories and mp3s + res.send( + JSON.stringify({ path:returnPath, contents:filesArray.concat(directories)}) + ); }); @@ -421,24 +371,16 @@ mstream.post('/download', function (req, res){ // ============================================================================ -// -// // Old way -// //const mstreamDB = require('./modules/database-'+program.databaseplugin+'.js'); -// // mstreamDB.setup(mstream, program.users, db); // TODO: ROOTDIR -// + // // New Way // // TODO: We need to pull this from the program var -var publicDBType = 'sqlite3'; // Can be sqlite3/mysql/LokiJS var dbSettings = program.database_plugin; const mstreamDB = require('./modules/db-management/database-master.js'); -// mstreamDB.setup(mstream, program.users, publicDBType, dbSettings); mstreamDB.setup(mstream, program); - // ============================================================================ - // TODO: Add individual song mstream.get('/db/add-songs', function(req, res){ res.send('Coming Soon!'); @@ -457,4 +399,5 @@ mstream.post( '/get-album-art', function(req, res){ // Start the server! +// TODO: Check if port is in use befoe firing up server const server = mstream.listen(program.port, function () {}); diff --git a/package.json b/package.json index e15fcf7..9675cd2 100644 --- a/package.json +++ b/package.json @@ -24,8 +24,10 @@ "jsonwebtoken": "^7.1.9", "musicmetadata": "^2.0.3", "my-local-ip": "^1.0.0", + "pouchdb": "^6.0.7", "slash": "^1.0.0", - "sqlite3": "^3.1.4" + "sqlite3": "^3.1.4", + "uuid": "^3.0.1" }, "optionalDependencies": { "nat-upnp": "^1.0.2", diff --git a/public/js/mstream.js b/public/js/mstream.js index ecd72f4..308a828 100755 --- a/public/js/mstream.js +++ b/public/js/mstream.js @@ -83,10 +83,9 @@ $(document).ready(function(){ request.done(function( msg ) { // Remove login screen - // TODO: set virtualDirectory - var decoded = $.parseJSON(msg); + // set virtualDirectory + var decoded = msg; virtualDirectory = decoded.vPath; - }); request.fail(function( jqXHR, textStatus ) { @@ -196,6 +195,9 @@ $(document).ready(function(){ function jPlayerSetMedia(fileLocation, filetype){ + if(virtualDirectory){ + fileLocation = virtualDirectory + '/' + fileLocation; + } document.getElementById("mplayer").setAttribute("src", fileLocation); document.getElementById("mplayer").setAttribute("title", fileLocation.split('/').pop()); } @@ -206,9 +208,6 @@ $(document).ready(function(){ function addFile2(that){ var filename = $(that).attr("id"); var file_location = $(that).data("file_location"); - if(virtualDirectory){ - file_location = virtualDirectory + '/' + file_location; - } if(accessKey){ file_location += '?token=' + accessKey; } diff --git a/untitled folder/exampleNewJSON.json b/untitled folder/exampleNewJSON.json index a7a7537..f4da2c4 100644 --- a/untitled folder/exampleNewJSON.json +++ b/untitled folder/exampleNewJSON.json @@ -1,29 +1,29 @@ { "port":3031, - "users":[ - { - "username":"paul", + "users":{ + "paul":{ "password":"glassjar98", "email":"paul@wall.com", "musicDir":"/path/to/music", - "guest":true, - "guestUsername":"guest name", - "guestPassword":"abcd", - "privateDB":"BEETS", "privateDBOptions":{ + "privateDB":"BEETS", "importDB":"path/to/sqlite3.db", - "beetspath":"/path/to/beets/music/dir" + "beetCommand":"Run This Command Update Call", + "quickSync": true, }, - "bannedGuestFunctions":"todo. make this optional" + "bannedGuestFunctions":"todo. make this optional", + "vPath":"UUID-GOES-HERE" }, - { - "username":"paul2", + "paul2":{ "password":"glassjar98", "email":"paul@wall2.com", "musicDir":"/path/to/music2", - "guest":"false", + }, + "paul2-guest":{ + "password":"glassjar98", + "guestTo":"paul2", } - ], + }, "userinterface":"public", "database_plugin":{ "type":"sqlite", @@ -38,6 +38,15 @@ "database_plugin-LOKI":{ "comming":"soon", }, - "secret":"SECRET GOES HERE. OR A PATH TO THE SECRET", - "tunnel":true, + "secret":"Secret for JSON web token", + "tunnel":{ + "gateway":"1.1.1.1", + "refreshInterval":10000, + "protocol":"upnp" + }, + "tunnelOptions":{}, + "salt":"Salt for passwords. optional. Necessarry forthe noHash function which allows hashed passwords to be stored in users array", + "noHash":true, + "ssl":"SSL OPTIONS" + } diff --git a/untitled folder/skeleton.json b/untitled folder/skeleton.json new file mode 100644 index 0000000..17b8c65 --- /dev/null +++ b/untitled folder/skeleton.json @@ -0,0 +1,4 @@ +{ + "port":5060, + "userinterface":"public", +} From 969c4fb1c12112dfdd0b5b3ec50f4e1a212986ed Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Wed, 14 Dec 2016 23:22:51 -0500 Subject: [PATCH 09/11] Fully functional DB, command line works again, improved beets functionality --- README.md | 115 ++++---- modules/configure-commander.js | 8 +- modules/configure-json-file.js | 7 +- .../db-management/database-beets-manager.js | 27 +- .../db-management/database-default-manager.js | 55 +++- modules/db-management/database-master.js | 151 +++++----- modules/db-read/database-public-beets.js | 264 ++++++++++++++++++ modules/db-write/database-default-sqlite.js | 21 +- untitled folder/JSONsqlite.json | 31 -- untitled folder/exampleNewJSON.json | 52 ---- untitled folder/new-db-model.txt | 60 ---- untitled folder/privatePublicDBModel.txt | 60 ---- untitled folder/skeleton.json | 4 - 13 files changed, 506 insertions(+), 349 deletions(-) create mode 100644 modules/db-read/database-public-beets.js delete mode 100644 untitled folder/JSONsqlite.json delete mode 100644 untitled folder/exampleNewJSON.json delete mode 100644 untitled folder/new-db-model.txt delete mode 100644 untitled folder/privatePublicDBModel.txt delete mode 100644 untitled folder/skeleton.json diff --git a/README.md b/README.md index 4250087..a407e4f 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ ## mStream mStream is an music streaming server written in NodeJS. It's focus is on ease of installation and FLAC streaming. mStream will work right out of the box without any configuration. -### Demo +#### Demo Check it out: http://darncoyotes.mstream.io/ - -### Main Features +#### Main Features * Supports FLAC streaming -* DB Plugin System. Choose the DB that best fits your needs +* Built in DB using SQLite. No need to run a separate DB * Works on Mac, Linux and Windows * [Integrates easily with Beets DB](https://github.com/beetbox/beets) * Allows multiple users @@ -15,47 +14,38 @@ Check it out: http://darncoyotes.mstream.io/ ## Installation -### Windows Executable - -There is work being done to port mStream to a Windows Executable. Check out the prototype here: -https://drive.google.com/file/d/0B1oiqEsIbjFidk8tVjR0TmZIb0k/view?usp=sharing - -### Default - +#### Dependencies mStream has the following dependencies: * NodeJS and NPM * Python 2 * GCC and G++ +* node-gyp -Once have all the dependencies you can install and setup mStream by doing the following - +#### Install on Ubuntu +Install NodeJS ```shell -npm install -g node-gyp -npm install -g mstream +curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash - +sudo-apt-get update +sudo apt-get install -y nodejs +``` + +Install GCC and node-gyp +```shell +sudo apt-get install -y build-essential +sudo npm install -g node-gyp +``` + +Install mStream +```shell +sudo npm install -g mstream cd /path/to/your/music - mstream ``` Make sure it's working by checking out http://localhost:3000/ - -### Install on Ubuntu -Copy and paste the following commands: - -```shell -curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash - -sudo apt-get install -y nodejs - -sudo apt-get install -y build-essential - -sudo npm install -g node-gyp -``` - - -### Using Docker -##### NOTE: This instructions are outdated and need to be updated +#### Using Docker Download the Dockerfile, or clone the repository, then run the following commands: @@ -75,57 +65,50 @@ docker run --rm -v /path/to/my/music:/music local/mstream -l -u username -x pass ## Usage -mStream can be configured by loading a JSON config file -```shell -mstream server.json -``` +mStream can be configured by using a JSON config file or by using flags in the command line. JSON config files are more flexible but more difficult to use. +This readme will not cover JSON config usage. See the examples folder to learn more. + +#### Set Port +```shell +mstream -p 5050 +``` ## User System - -The current user system is a simple as it comes. There are two users you can have, main and guest. Guest users do not have any access to API functions that write to the file system. Currently guest users cannot access the save-playlist, recursive-scan, or delete-playlist functions +mStream can have a single user and guest when being setup using the command line. ```shell -mstream -l -u [username] -x [password] +# Set User +mstream -u [username] -x [password] +# Set user and guest +mstream -u [username] -x [password] -G [guest name] -X [guest password] ``` -The user system is simple for a few reasons. First, I wanted to have a user system that doesn't need a database to work. Secondly, mStream is a personal server and most users don't need anything more complex than this. - +Multiple users can be set using JSON config files ## Database Options +mStream uses sqlite by default. You can either use mStream's default database [or tap into BeetsDB](https://github.com/beetbox/beets) -mSTream's datbase will work right out of the box without +#### Beets DB -### Database Plugin System +```shell +mstream -D beets -d /path/to/beets.db +``` -mStream 2.0 is written so that you can choose what DB system you use. Currently only sqlite is supported, but in the future their will be more options: - -- SQLite -- MySQL -- PouchDB: A NoSQL alternative +When using Beets, mStream is put into a read only mode. mStream will not be able to write to any tables that are managed by Beets. Playlist functionality is not affected by this since playlists are stored in a separate table. +#### Built In DB +mStream can read metadata and write it's own database. By default mstream will create a database in the folder it's launched in called 'mstreamdb.lite'. You can manually set the databse file with: - -### Import DB -http://beets.io/ - -User's can choose how their files are managed. By default mstream will manage the user's DB. User's also have the option to import their DB from somewhere else - -#### beets DB - -#### use mStream to build your DB - -Use the /db/recursive-scan API call to kickoff a full scan of your library. Currently this is the only way to add files to the library. Version 2 of mStream will include new functions to update the library more efficiently - - - +```shell +mstream -d /path/to/mstream.db +``` ## Automatically setup port forwarding -#### Please note that this feature is still experimental mStream can try to automatically open a port to the internet. Use the '-t' command to try to setup port forwarding. Additionally you can use the '-g' command to set the gateway IP manually. If you don't include '-g', the program will use an extension to try to figure it out @@ -140,6 +123,12 @@ mstream musicDirectory/ -t -g 192.168.1.1 Please note that not all routers will allow this. Some routers may close this port after a period of time. +You can get around this by having mStream retry this on a regular interval + +``` +mstream -t -r [time in milliseconds] +mstream -t -r 10000 +``` ## Known Issues diff --git a/modules/configure-commander.js b/modules/configure-commander.js index 8fe60cd..7436f3d 100644 --- a/modules/configure-commander.js +++ b/modules/configure-commander.js @@ -1,12 +1,12 @@ exports.setup = function(args){ const program = require('commander'); program - .version('1.21.0') + .version('2.0.0') .option('-p, --port ', 'Select Port', /^\d+$/i, 3000) .option('-t, --tunnel', 'Use nat-pmp to configure port fowarding') .option('-g, --gateway ', 'Manually set gateway IP for the tunnel option') .option('-r, --refresh ', 'Refresh rate', /^\d+$/i) - .option('-o, --protocol ', 'Refresh rate', /^\d+$/i) + .option('-o, --protocol ', 'Protocol for tunneling', /^(upnp|natpnp)$/i, 'natpnp') .option('-u, --user ', 'Set Username') .option('-x, --password ', 'Set Password') .option('-e, --email ', 'Set User Email (optional)') @@ -15,6 +15,7 @@ exports.setup = function(args){ .option('-d, --database ', 'Specify Database Filepath', 'mstreamdb.lite') .option('-i, --userinterface ', 'Specify folder name that will be served as the UI', 'public') .option('-s, --secret ', 'Set the login secret key') + .option('-D, --databaseplugin ', '', /^(sqlite|beets)$/i, 'sqlite') // TODO: Add support for other DBs when ready .parse(args); @@ -53,10 +54,9 @@ exports.setup = function(args){ // db plugins program3.database_plugin = { - type:"sqlite", + type:program.databaseplugin, dbPath:program.database }; - // TODO: Add support for other DBs when ready // port forwarding if(program.tunnel){ diff --git a/modules/configure-json-file.js b/modules/configure-json-file.js index e85603d..db00e8e 100644 --- a/modules/configure-json-file.js +++ b/modules/configure-json-file.js @@ -52,9 +52,10 @@ exports.setup = function(args, rootDir){ } // TODO: Preform a full range of checks - - if(!isInt(loadJson.tunnel.refreshInterval)){ - return {error:"Refresh interval must be an integer"}; + if(loadJson.tunnel){ + if(loadJson.tunnel.refreshInterval && !isInt(loadJson.tunnel.refreshInterval)){ + return {error:"Refresh interval must be an integer"}; + } } // Export JSON diff --git a/modules/db-management/database-beets-manager.js b/modules/db-management/database-beets-manager.js index 0f56db5..69c3c13 100644 --- a/modules/db-management/database-beets-manager.js +++ b/modules/db-management/database-beets-manager.js @@ -38,16 +38,19 @@ if(loadJson.dbSettings.type == 'sqlite'){ run(); -// TODO: It might be worth combing this into the default-sqlite files since they share some functions // SELECT * FROM items LEFT JOIN item_attributes ON // item_attributes.entity_id = items.id // AND item_attributes.key = 'checksum' // GROUP BY items.id, item_attributes.key; function run(){ + let sql = "SELECT * FROM items LEFT JOIN item_attributes ON item_attributes.entity_id = items.id AND item_attributes.key = 'checksum' GROUP BY items.id, item_attributes.key;"; beetsDB.all(sql, function(err, files){ files = dbPublic.reformatData(files); + + // TODO: We can make this more efficient by comparing the differences and just adding/deleting the changes dbPublic.purgeDB(username); + dbPublic.addToDB(files); if(loadJson.privateDBOptions.quickSync === false){ @@ -57,6 +60,28 @@ function run(){ } +function insertEntries(numberToInsert = 99, loopToEnd = false){ + var insertThese = []; + + while(insertThese.length != numberToInsert ){ + if(arrayOfSongs.length == 0){ + break; + } + insertThese.push(arrayOfSongs.pop()); + } + + dbRead.insertEntries(insertThese, loadJson.username, function(){ + // Recursivly run this function until all songs have been added + if(loopToEnd && arrayOfSongs.length != 0){ + insertEntries(numberToInsert, true); + }else{ + // For the generator + parseFilesGenerator.next(); + } + }); +} + + // TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO:TODO: function smokeThatHash( blazeItEveryDay = false){ diff --git a/modules/db-management/database-default-manager.js b/modules/db-management/database-default-manager.js index 0c86361..ab14ed2 100644 --- a/modules/db-management/database-default-manager.js +++ b/modules/db-management/database-default-manager.js @@ -74,11 +74,16 @@ function *rescanAllDirectories(directoryToScan){ // process all updated files while(filesToProcess.updatedFiles.length > 0) { - // TODO: Handle Editted songs - //yield hashOneUpdatedSong(filesToProcess.updatedFiles.pop()); + // Handle Editted songs + yield parseUpdatedSong(filesToProcess.updatedFiles.pop()); + yield insertEntries(50, true); } // TODO: Process deleted files + while(filesToProcess.deletedFiles.length > 0) { + // Handle Editted songs + yield deleteFile(filesToProcess.deletedFiles.pop()); + } // Exit process.exit(0); @@ -193,8 +198,54 @@ function rescanDirectory(dir){ } +// TODO: Fix this +function parseUpdatedSong(filePath){ + // Check sha256 hash and confirm it has changed + // Update file status in DB accordingly + var fileStream = fs.createReadStream(filePath); + + var hash = crypto.createHash('sha256'); + hash.setEncoding('hex'); + + + fileStream.on('end', function () { + hash.end(); + + var hashIt = String(hash.read()); + + // compare hashes + //db.all("SELECT * FROM files WHERE path=? AND hash=?", [filePath, hashIt], function(err, rows){ + dbRead.getHashedEntry(hashIt, filePath, loadJson.username, function(rows){ + console.log(rows); + // No match found, file needs to be updated + if( !rows || rows.length === 0 ){ + // TODO: delete entry + dbRead.deleteFile(filePath, loadJson.username, function(){ + // Re-add entry + parseFile(filePath); + }); + + }else{ + parseFilesGenerator.next(); + + } + + }); + }); + + fileStream.pipe(hash); +} + + +function deleteFile(filepath){ + dbRead.deleteFile(filepath, loadJson.username, function(){ + // Re-add entry + parseFilesGenerator.next(); + }); +} + function parseFile(thisSong){ var filestat = fs.statSync(thisSong); if(!filestat.isFile()){ diff --git a/modules/db-management/database-master.js b/modules/db-management/database-master.js index f320b5a..4ec7142 100644 --- a/modules/db-management/database-master.js +++ b/modules/db-management/database-master.js @@ -14,81 +14,85 @@ exports.setup = function(mstream, program){ var userDBStatus = {}; + // Certain setups are readonly + if(program.database_plugin.type != 'beets'){ - mstream.get('/db/recursive-scan', function(req,res){ - // Check if user is already being scanned - if(userDBStatus[req.user.username] == true){ - res.send('In Process. Please check status.'); - return; - } - // - userDBStatus[req.user.username] = true; + mstream.get('/db/recursive-scan', function(req,res){ + // Check if user is already being scanned + if(userDBStatus[req.user.username] == true){ + res.send('In Process. Please check status.'); + return; + } + // + userDBStatus[req.user.username] = true; - // Get user's db setup - if(!req.user.privateDB || req.user.privateDB == 'DEFAULT'){ - forkDefault(req.user, program.database_plugin); - res.send('IT\'S HAPPENING!'); - return; - } + // Get user's db setup + if(!req.user.privateDB || req.user.privateDB == 'DEFAULT'){ + forkDefault(req.user, program.database_plugin); + res.send('IT\'S HAPPENING!'); + return; + } - if(req.user.privateDB == 'BEETS'){ - forkBeets(req.user); - res.send('IT\'S HAPPENING! \n NOW WITH 60% MORE BEETS!'); - return; - } + if(req.user.privateDB == 'BEETS'){ + forkBeets(req.user); + res.send('IT\'S HAPPENING! \n NOW WITH 60% MORE BEETS!'); + return; + } - // YOUR CONFIG IS BAD AND YOU SHOULD FEEL BAD - userDBStatus[req.user.username] = false; - res.send('YOUR CONFIG IS BAD AND YOU SHOULD FEEL BAD. ABORTING!'); + // YOUR CONFIG IS BAD AND YOU SHOULD FEEL BAD + userDBStatus[req.user.username] = false; + res.send('YOUR CONFIG IS BAD AND YOU SHOULD FEEL BAD. ABORTING!'); - }); - - - /////////////////////////// - // TODO: Should we have a API call that can kill any process associated with a user and reset their scan value to false? - /////////////////////////// - - /////////////////////////// - // TODO: We could use some kind of manager to make sure we don't spawn to many child processes - // For now we spawn indiscriminately and let the CPU sort it out - /////////////////////////// - - // TODO: Fill this out - function forkBeets(user, publicDBType, dbSettings){ - // Pull beets commands from config - - // Run commands - // beet import -A --group-albums /path/to/music - // beet check -a - // find ~ -type d -empty -delete - } - - function forkDefault(user, dbSettings){ - // TODO: IMPLEMENT FORK PROPERLY - // SEND JSON DATA TO WORKER PROCESS - var jsonLoad = { - username:user.username, - userDir:user.musicDir, - dbSettings:dbSettings - } - - const forkedScan = child.fork(__dirname + '/database-default-manager.js', [JSON.stringify(jsonLoad)]); - - // forkedScan.stdout.on('data', (data) => { - // console.log(`stdout: ${data}`); - // }); - // - // forkedScan.stderr.on('data', (data) => { - // console.log(`stderr: ${data}`); - // }); - - forkedScan.on('close', (code) => { - userDBStatus[user.username] = false; - console.log(`child process exited with code ${code}`); }); - } + /////////////////////////// + // TODO: Should we have a API call that can kill any process associated with a user and reset their scan value to false? + /////////////////////////// + + /////////////////////////// + // TODO: We could use some kind of manager to make sure we don't spawn to many child processes + // For now we spawn indiscriminately and let the CPU sort it out + /////////////////////////// + + // TODO: Fill this out + function forkBeets(user, publicDBType, dbSettings){ + // Pull beets commands from config + + // Run commands + // beet import -A --group-albums /path/to/music + // beet check -a + // find ~ -type d -empty -delete + } + + function forkDefault(user, dbSettings){ + // TODO: IMPLEMENT FORK PROPERLY + // SEND JSON DATA TO WORKER PROCESS + var jsonLoad = { + username:user.username, + userDir:user.musicDir, + dbSettings:dbSettings + } + + const forkedScan = child.fork(__dirname + '/database-default-manager.js', [JSON.stringify(jsonLoad)]); + + // forkedScan.stdout.on('data', (data) => { + // console.log(`stdout: ${data}`); + // }); + // + // forkedScan.stderr.on('data', (data) => { + // console.log(`stderr: ${data}`); + // }); + + forkedScan.on('close', (code) => { + userDBStatus[user.username] = false; + console.log(`child process exited with code ${code}`); + }); + } + + + } + // TODO: Special function that just transfers fiels from users private DB to public DB mstream.get('/db/import-DB', function(req,res){ // Get user info @@ -96,6 +100,17 @@ exports.setup = function(mstream, program){ // Return if user is not using private DB // Delete users files // Pull all files from DB and add to publicDB + + if(program.database_plugin.beets_command){ + // TODO: Run Command + }else if(req.user.privateDBOptions.beets_command){ + // TODO: Run Command + }else{ + // res.send('NO GO'); + } + + res.send('Coming Soon!'); + }); @@ -106,7 +121,7 @@ exports.setup = function(mstream, program){ - + // TODO: Modify this to use the public DB mstream.get('/db/download-db', function(req, res){ // Check user for beets db if(!req.user.privateDB || req.user.privateDB != 'BEETS'){ diff --git a/modules/db-read/database-public-beets.js b/modules/db-read/database-public-beets.js new file mode 100644 index 0000000..abdacaa --- /dev/null +++ b/modules/db-read/database-public-beets.js @@ -0,0 +1,264 @@ +const sqlite3 = require('sqlite3').verbose(); +const slash = require('slash'); +const fe = require('path'); +const crypto = require('crypto'); + + + +// function that takes in a json array of songs and saves them to the sqlite db + // must contain the username and filepath for each song + +// function that gets artist info and returns json array of albums +// function that searches db and returns json array of albums and artists +// function that takes ina playlist name and searchs db for that playlist and returns a json array of songs for that playlist +// BASICALLY, all the functions we have no but de-couple them from the Express API calls + + +// TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: TODO: +// TODO: Pass in user music dir and compare it to the path using LIKE +// This way If the user is set to use a sub folder of the DB folder, we can ignore all folders above what the user has permission for + + +function getFileType(filename){ + return filename.split(".").pop(); +} + + +exports.setup = function(mstream, dbSettings){ + const db = new sqlite3.Database(dbSettings.dbPath); + + + // Create a playlist table + db.run("CREATE TABLE IF NOT EXISTS mstream_playlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, playlist_name varchar, filepath varchar, hide int DEFAULT 0, user VARCHAR, created datetime default current_timestamp);", function() { + }); + + + + // TODO: This needs to be tested to see if it works on extra large playlists (think thousands of entries) + // TODO: Ban saving playlists that are > 10,000 items long + mstream.post('/saveplaylist', function (req, res){ + var title = req.body.title; + var songs = req.body.stuff; + + // Check if this playlist already exists + db.all("SELECT id FROM mstream_playlists WHERE playlist_name = ?", [title], function(err, rows) { + + db.serialize(function() { + + // We need to delete anys existing entries + if(rows && rows.length > 0){ + db.run("DELETE FROM mstream_playlists WHERE playlist_name = ?;", [title]); + } + + // Now we add the new entries + var sql2 = "insert into mstream_playlists (playlist_name, filepath) values "; + var sqlParser = []; + + while(songs.length > 0) { + var song = songs.shift(); + + sql2 += "(?, ?), "; + sqlParser.push(title); + // TODO: We need to strip out the vPath + // We need to allow pre-set vPaths in the config file + // Then strip out the vPath here + sqlParser.push( fe.join(req.user.musicDir, song) ); + + } + + sql2 = sql2.slice(0, -2); + sql2 += ";"; + + db.run(sql2, sqlParser, function(){ + res.send('DONE'); + }); + + }); + }); + }); + + + // Attach API calls to functions + mstream.get('/getallplaylists', function (req, res){ + // TODO: In V2 we need to change this to ignore hidden playlists + // TODO: db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE hide=0;", function(err, rows){ + db.all("SELECT DISTINCT playlist_name FROM mstream_playlists", function(err, rows){ + var playlists = []; + + // loop through files + for (var i = 0; i < rows.length; i++) { + if(rows[i].playlist_name){ + playlists.push({name: rows[i].playlist_name}); + } + } + + res.send(JSON.stringify(playlists)); + }); + }); + mstream.get('/loadplaylist', function (req, res){ + var playlist = req.query.playlistname; + + db.all("SELECT * FROM mstream_playlists WHERE playlist_name = ? ORDER BY id COLLATE NOCASE ASC", [playlist], function(err, rows){ + var returnThis = []; + + for (var i = 0; i < rows.length; i++) { + + // var tempName = rows[i].filepath.split('/').slice(-1)[0]; + var tempName = fe.basename(rows[i].filepath); + var extension = getFileType(rows[i].filepath); + var filepath = slash(fe.relative(req.user.musicDir, rows[i].filepath)); + console.log(filepath); + returnThis.push({name: tempName, file: filepath, filetype: extension }); + } + + res.send(JSON.stringify(returnThis)); + }); + }); + mstream.get('/deleteplaylist', function(req, res){ + var playlistname = req.query.playlistname; + + // Handle a soft delete + if(req.query.hide && parseInt(req.query.hide) === 1 ){ + db.run("UPDATE mstream_playlists SET hide = 1 WHERE playlist_name = ?;", [playlistname], function(){ + res.send('DONE'); + + }); + }else{ // Permentaly delete + + // Delete playlist from DB + db.run("DELETE FROM mstream_playlists WHERE playlist_name = ?;", [playlistname], function(){ + res.send('DONE'); + + }); + } + }); + + + mstream.post('/db/search', function(req, res){ + var searchTerm = "%" + req.body.search + "%" ; + + var returnThis = {"albums":[], "artists":[]}; + + // TODO: Combine SQL calls into one + db.serialize(function() { + + var sqlAlbum = "SELECT DISTINCT album FROM items WHERE items.album LIKE ? ORDER BY album COLLATE NOCASE ASC;"; + db.all(sqlAlbum, [searchTerm], function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + for (var i = 0; i < rows.length; i++) { + if(rows[i].album){ + returnThis.albums.push(rows[i].album); + } + } + }); + + + var sqlArtist = "SELECT DISTINCT artist FROM items WHERE items.artist LIKE ? ORDER BY artist COLLATE NOCASE ASC;"; + db.all(sqlArtist, [searchTerm], function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + for (var i = 0; i < rows.length; i++) { + if(rows[i].artist){ + returnThis.artists.push(rows[i].artist); + } + } + + res.send(JSON.stringify(returnThis)); + }); + }); + }); + + mstream.get('/db/artists', function (req, res) { + var artists = {"artists":[]}; + + var sql = "SELECT DISTINCT artist FROM items ORDER BY artist COLLATE NOCASE ASC;"; + db.all(sql, function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + var returnArray = []; + for (var i = 0; i < rows.length; i++) { + if(rows[i].artist){ + // rows.splice(i, 1); + artists.artists.push(rows[i].artist); + } + } + + res.send(JSON.stringify(artists)); + }); + }); + + mstream.post('/db/artists-albums', function (req, res) { + var albums = {"albums":[]}; + + // TODO: Make a list of all songs without null albums and add them to the response + var sql = "SELECT DISTINCT album FROM items WHERE artist = ? ORDER BY album COLLATE NOCASE ASC;"; + db.all(sql, [req.body.artist], function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + var returnArray = []; + for (var i = 0; i < rows.length; i++) { + if(rows[i].album){ + albums.albums.push(rows[i].album); + } + } + + res.send(JSON.stringify(albums)); + }); + }); + + mstream.get('/db/albums', function (req, res) { + var albums = {"albums":[]}; + + var sql = "SELECT DISTINCT album FROM items ORDER BY album COLLATE NOCASE ASC;"; + db.all(sql, function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + var returnArray = []; + for (var i = 0; i < rows.length; i++) { + if(rows[i].album){ + albums.albums.push(rows[i].album); + } + } + + res.send(JSON.stringify(albums)); + }); + }); + + mstream.post('/db/album-songs', function (req, res) { + var sql = "SELECT title, artist, album, format, year, cast(path as TEXT), track FROM items WHERE album = ? ORDER BY track ASC;"; + db.all(sql, [req.body.album], function(err, rows) { + if(err){ + res.status(500).json({ error: 'DB Error' }); + return; + } + + // Format data for API + for(var i in rows ){ + var path = String(rows[i]['cast(path as TEXT)']); + + rows[i].format = rows[i].format.toLowerCase(); // make sure the format is lowecase + rows[i].file_location = slash(fe.relative(req.user.musicDir, path)); // Get the local file location + rows[i].filename = fe.basename( path ); // Ge the filname + } + + res.send(JSON.stringify(rows)); + }); + }); + +} diff --git a/modules/db-write/database-default-sqlite.js b/modules/db-write/database-default-sqlite.js index 661b553..2b5c7fa 100644 --- a/modules/db-write/database-default-sqlite.js +++ b/modules/db-write/database-default-sqlite.js @@ -77,8 +77,9 @@ exports.insertEntries = function(arrayOfSongs, username, callback){ // Function that reformats data from beets +// TODO: Fix this exports.reformatData = function(username){ - let sql = "DELETE FROM items WHERE user = ?"; + let sql = "DELETE FROM items WHERE user = ?;"; db.all(sql, username, function(err, rows){ }); } @@ -87,3 +88,21 @@ exports.reformatData = function(username){ exports.purgeDB = function(){ } + + +exports.deleteFile = function(path, user, callback){ + let sql = "DELETE FROM items WHERE path = ? AND user = ?;"; + db.run(sql, [path, user], function() { + console.log('ITS DONE'); + callback(); + }); + +} + +exports.getHashedEntry = function(hash, path, user, callback){ + db.all("SELECT path, hash FROM items WHERE hash = ? AND path = ? AND user = ?;", [hash, path, user], function(err, rows){ + console.log(rows); + // callback function + callback(rows); + }); +} diff --git a/untitled folder/JSONsqlite.json b/untitled folder/JSONsqlite.json deleted file mode 100644 index 896d9a9..0000000 --- a/untitled folder/JSONsqlite.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "port":3031, - "users":[ - { - "username":"paul", - "password":"glassjar98", - "email":"paul@wall.com", - "musicDir":"/path/to/music", - "guest":true, - "guestUsername":"guest name", - "guestPassword":"abcd", - "privateDB":"BEETS", - "privateDBOptions":{ - "importDB":"path/to/sqlite3.db", - "beetspath":"/path/to/beets/music/dir" - }, - "bannedGuestFunctions":"todo. make this optional" - }, - { - "username":"paul2", - "password":"glassjar98", - "email":"paul@wall2.com", - "musicDir":"/path/to/music2", - } - ], - "userinterface":"public", - "database_plugin":{ - "type":"sqlite", - "dbPath":"/path/to/db", - } -} diff --git a/untitled folder/exampleNewJSON.json b/untitled folder/exampleNewJSON.json deleted file mode 100644 index f4da2c4..0000000 --- a/untitled folder/exampleNewJSON.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "port":3031, - "users":{ - "paul":{ - "password":"glassjar98", - "email":"paul@wall.com", - "musicDir":"/path/to/music", - "privateDBOptions":{ - "privateDB":"BEETS", - "importDB":"path/to/sqlite3.db", - "beetCommand":"Run This Command Update Call", - "quickSync": true, - }, - "bannedGuestFunctions":"todo. make this optional", - "vPath":"UUID-GOES-HERE" - }, - "paul2":{ - "password":"glassjar98", - "email":"paul@wall2.com", - "musicDir":"/path/to/music2", - }, - "paul2-guest":{ - "password":"glassjar98", - "guestTo":"paul2", - } - }, - "userinterface":"public", - "database_plugin":{ - "type":"sqlite", - "dbPath":"/path/to/db", - }, - "database_plugin-MYSQL":{ - "type":"mysql", - "username":"lol", - "password":"lol", - "db-name":"lolDB" - }, - "database_plugin-LOKI":{ - "comming":"soon", - }, - "secret":"Secret for JSON web token", - "tunnel":{ - "gateway":"1.1.1.1", - "refreshInterval":10000, - "protocol":"upnp" - }, - "tunnelOptions":{}, - "salt":"Salt for passwords. optional. Necessarry forthe noHash function which allows hashed passwords to be stored in users array", - "noHash":true, - "ssl":"SSL OPTIONS" - -} diff --git a/untitled folder/new-db-model.txt b/untitled folder/new-db-model.txt deleted file mode 100644 index 9b4193d..0000000 --- a/untitled folder/new-db-model.txt +++ /dev/null @@ -1,60 +0,0 @@ -UUSER BASED SETINGS -beets-management : true/false -beets-import-path: path where beets gets it's music from -local-user-db : db to port music in from -single-user-backup : true / false - Backs up users files to a private db. Will auto to false if user is using beets - - -GLOBAL SETTINGS -public-db-type: sqlite3, mysql, other - -SHOULD WE MAKE BEETS MANAGEMENT A GLOBAL THING??? - - - - - - -========================================================================================== -The options are: - -build and manage everything on a public scale and ignore all private DBs - This is how mStream Express will do things - -Build everything on a public scale and backup on a private scale (when requested) - // This might be able to be ignored for the first revision - -TODO: Mix beets and no-beets users - -Build eveything on a private scale using beets, and then build a public DB based on that - - - -========================================================================================== - - -- Public DB management plugin system - - input: db type - sqlite, mysql, etc - - input: users array - - input: mstream variable - - loads the appropriate db plugin based on input - - db plugins contain all API calls that construct the DB (playlists, db search, albums, artits, etc) - - (OPTIONAL: Include a NO DB option that saves playlists as files and nothing else) - - Handles all individual user db settings (3 options) - - Beets management - - mstream management - no private DB - - mstream management - private db backup - - holds functions that can manage beets, scan beets for updates and update local DB, and make backups in different db types - - routes user's to appropriate functions - - need to separate the api endpoint functions, the db functions, and the routing that connects the two - - can optionally not load some functions if no users are using them - - - -======================================================================================================== - -NOTES -- all things are going to be based on usernames. - - Should make functions that can generate and save json. - - This way we can have a command that adds and saves a new user diff --git a/untitled folder/privatePublicDBModel.txt b/untitled folder/privatePublicDBModel.txt deleted file mode 100644 index 716c6d6..0000000 --- a/untitled folder/privatePublicDBModel.txt +++ /dev/null @@ -1,60 +0,0 @@ -Multi user - -make all user music folders available seperatly - -Make one DB that stores everything - Make seperate libraries that can import from different DBs - - -Need a seperate table that logs update times - - -==================================================================================== -Beets DB - Create and store hash of each private DB, and a last updated timestamp - - Compare DB hashes - If different, scan private DB for fields added after timestamp - Add all new fields to public DB - - -Built In DB - Spawn new process - Process all newly added files - Add new files to both private user DB and public shared DB - - - -================================================================================ -The Private/Public DB model - -Each user stores a private sqlite3DB (or maybe something else) -When a filescan is done, both the private DB is updated -An mstream plugin can read the private DB and insert it into a shared public DB - public DBs can easily be created and destroyed to change expanding server needs - - - -HOSTING -Private DBs are mamaged by beets -Public DBs are managed by MySQL - -MSTREAM EXPRESS -NO Private DBS -All users files are managed in one big public DB - - - -==================================================================================== -All Scenarios: - -User is using mstream for everything -sqlite for public and private db - -User is is using sqlite3 for public DB, and beets for private database - -Mysql for public DB, beets for private db - -Mysql for public and private - -Bonus: Custom javascript DB (ex: http://lokijs.org/#/) for everything diff --git a/untitled folder/skeleton.json b/untitled folder/skeleton.json deleted file mode 100644 index 17b8c65..0000000 --- a/untitled folder/skeleton.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "port":5060, - "userinterface":"public", -} From 473ebfa859d0c27ae10cb1c1ba4343ebd70cc00b Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Wed, 14 Dec 2016 23:44:40 -0500 Subject: [PATCH 10/11] Example json file --- exampleJSON/allOptions.json | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 exampleJSON/allOptions.json diff --git a/exampleJSON/allOptions.json b/exampleJSON/allOptions.json new file mode 100644 index 0000000..518a7d3 --- /dev/null +++ b/exampleJSON/allOptions.json @@ -0,0 +1,28 @@ +{ + "port":3031, + "users":{ + "paul":{ + "password":"glassjar98", + "email":"paul@wall.com", + "musicDir":"/path/to/music", + }, + "paul2":{ + "password":"glassjar98", + "email":"paul@wall2.com", + "musicDir":"/path/to/music2", + }, + "paul2-guest":{ + "password":"glassjar98", + "guestTo":"paul2", + } + }, + "userinterface":"public", + "database_plugin":{ + "type":"sqlite", + "dbPath":"/path/to/db", + }, + "tunnel":{ + "refreshInterval":10000, + "protocol":"upnp" + } +} From 00c3de4a574a47815e7a767481bba4a22c13825e Mon Sep 17 00:00:00 2001 From: Paul Sori Date: Wed, 14 Dec 2016 23:45:41 -0500 Subject: [PATCH 11/11] 2.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9675cd2..3b86ff6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mstream", - "version": "1.28.0", + "version": "2.0.0", "description": "music streaming server", "main": "mstream.js", "bin": {