diff --git a/modules/db-management/database-default-manager.js b/modules/db-management/database-default-manager.js index 82a876e..8046b80 100644 --- a/modules/db-management/database-default-manager.js +++ b/modules/db-management/database-default-manager.js @@ -26,10 +26,7 @@ const crypto = require('crypto'); // Setup DB layer // The DB functions are dcoupled from this so they can easily be swapped out -const dbRead = require('../db-write/database-default-'+loadJson.dbSettings.type+'.js'); -if(loadJson.dbSettings.type == 'sqlite'){ - dbRead.setup(loadJson.dbSettings.dbPath); -} +const dbRead = require('../db-write/database-default-loki.js'); // Global Vars var globalCurrentFileList = {}; // Map of file paths to metadata @@ -42,22 +39,29 @@ parseFilesGenerator.next(); // Scan the directory for new, modified, and deleted files function *rescanAllDirectories(directoryToScan){ + yield dbRead.setup(loadJson.dbSettings.dbPath, function(){ + parseFilesGenerator.next(); + }); // Pull filelist from DB - yield pullFromDB(); + pullFromDB(); // Loop through current files and compare them to the files pulled from the DB recursiveScan(directoryToScan); // Delete Files for (var i=0; i < listOfFilesToDelete.length; i++) { - yield deleteFile(listOfFilesToDelete[i]); + deleteFile(listOfFilesToDelete[i]); } // Delete all remaining files for (var file in globalCurrentFileList) { - yield deleteFile(file); + deleteFile(file); } // Parse and add files to DB for (var i=0; i < listOfFilesToParse.length; i++) { yield parseFile(listOfFilesToParse[i]); } + + yield dbRead.savedb(function(){ + parseFilesGenerator.next(); + }) // Exit process.exit(0); } @@ -66,10 +70,8 @@ function *rescanAllDirectories(directoryToScan){ function pullFromDB(){ dbRead.getUserFiles(loadJson, function(rows){ for(var s of rows){ - globalCurrentFileList[s.path] = s; + globalCurrentFileList[s.filepath] = s; } - // Go to next step - parseFilesGenerator.next(); }); } @@ -105,7 +107,7 @@ function recursiveScan(dir, fileTypesArray){ } // check the file_modified_date - if(stat.mtime.getTime() !== globalCurrentFileList[filepath].file_modified_date){ + if(stat.mtime.getTime() !== globalCurrentFileList[filepath].modified){ listOfFilesToParse.push(filepath); listOfFilesToDelete.push(filepath); } @@ -119,7 +121,6 @@ function recursiveScan(dir, fileTypesArray){ function parseFile(thisSong){ - // console.log(thisSong); var filestat = fs.statSync(thisSong); if(!filestat.isFile()){ // TODO: Something is fucky, log it @@ -161,7 +162,6 @@ function calculateHash (thisSong, songInfo) { if (songInfo.picture && songInfo.picture[0]) { bufferString = songInfo.picture[0].data.toString('utf8'); picFormat = songInfo.picture[0].format; - // console.log(songInfo.picture); } else if (false) { // TODO: Check if there is album art in base folder } @@ -197,6 +197,7 @@ function calculateHash (thisSong, songInfo) { } function deleteFile(filepath){ + console.log(filepath) dbRead.deleteFile(filepath, loadJson.username, function(){ // Re-add entry parseFilesGenerator.next(); diff --git a/modules/db-management/database-master.js b/modules/db-management/database-master.js index 69b48dc..87bd546 100644 --- a/modules/db-management/database-master.js +++ b/modules/db-management/database-master.js @@ -4,7 +4,7 @@ exports.setup = function(mstream, program){ // Load in API enndpoints // TODO: Change the name of this file - const mstreamReadPublicDB = require('../db-read/database-public-sqlite.js'); + const mstreamReadPublicDB = require('../db-read/database-public-loki.js'); mstreamReadPublicDB.setup(mstream, program.database_plugin); // Var that keeps track of DB scans going on @@ -225,7 +225,7 @@ exports.setup = function(mstream, program){ username: username, musicDir: program.users[username].musicDir, }, function(){ - // TODO: Add generator and yield here + mstreamReadPublicDB.loadDB(); bootScanGenerator.next(); }); } diff --git a/modules/db-read/database-public-loki.js b/modules/db-read/database-public-loki.js new file mode 100644 index 0000000..52f4b92 --- /dev/null +++ b/modules/db-read/database-public-loki.js @@ -0,0 +1,263 @@ +// TODO: This style looks up things synchronously from lokijs +// ideally we would ru na loki server on a seperate thread so we don't block our server + +const fe = require('path') +const crypto = require('crypto') + +// These functions will take in JSON arrays of song data and then save that dat to the DB +const loki = require('lokijs') +const filesdb = new loki('files.db') + +var fileCollection +var playlistColection + + + +function loadDB(){ + filesdb.loadDatabase({}, function(err) { + if (err) { + console.log("error : " + err); + } + else { + console.log("database loaded XXX"); + } + // Add a collection to the database + fileCollection = filesdb.getCollection('files') + playlistColection = filesdb.getCollection('playlists') + + }); +} + +// Load DB on boot +loadDB(); + +exports.loadDB = function(){ + loadDB(); +} + +function getFileType(filename){ + return filename.split(".").pop() +} + +exports.getNumberOfFiles = function(username, callback){ + var results = fileCollection.count({ 'user': username }) + callback(results) +} + +exports.setup = function (mstream, dbSettings){ + // Metadata lookup + mstream.post('/db/metadata', function (req, res){ + var relativePath = req.body.filepath; + var fullpath = fe.join(req.user.musicDir, relativePath); + + var result = fileCollection.findOne({'filepath': fullpath}); + res.json({ + "filepath":relativePath, + "metadata":{ + "artist":result.artist, + "hash": result.hash, + "album":result.album, + "track":result.track, + "title":result.title, + "year":result.year, + "album-art":row.albumArtFilename + } + }); + }); + + + // TODO: This needs to be tested to see if it works on extra large playlists (think thousands of entries) + // TODO: Ban saving playlists that are > 10,000 items long + mstream.post('/playlist/save', function (req, res){ + var title = req.body.title; + var songs = req.body.songs; + + // Check if this playlist already exists + db.all("SELECT id FROM mstream_playlists WHERE playlist_name = ? AND user = ?;", [title, req.user.username], function(err, rows) { + db.serialize(function() { + // We need to delete anys existing entries + if(rows && rows.length > 0){ + db.run("DELETE FROM mstream_playlists WHERE playlist_name = ? AND user = ?;", [title, req.user.username]); + } + + // Now we add the new entries + var sql2 = "insert into mstream_playlists (playlist_name, filepath, user) values "; + var sqlParser = []; + + while(songs.length > 0) { + var song = songs.shift(); + + sql2 += "(?, ?, ?), "; + sqlParser.push(title); + sqlParser.push( fe.join(req.user.musicDir, song) ); + sqlParser.push( req.user.username ); + } + + sql2 = sql2.slice(0, -2); + sql2 += ";"; + + db.run(sql2, sqlParser, function(){ + res.json({success: true}); + }); + + }); + }); + }); + + + // Attach API calls to functions + mstream.get('/playlist/getall', function (req, res){ + // TODO: In V2 we need to change this to ignore hidden playlists + // TODO: db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE hide=0;", function(err, rows){ + db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE user = ?", [req.user.username], function(err, rows){ + var playlists = []; + + // loop through files + for (var i = 0; i < rows.length; i++) { + if(rows[i].playlist_name){ + playlists.push({name: rows[i].playlist_name}); + } + } + + res.json(playlists); + }); + }); + + mstream.post('/playlist/load', function (req, res){ + var playlist = req.body.playlistname; + + db.all("SELECT * FROM mstream_playlists WHERE playlist_name = ? AND user = ? ORDER BY id COLLATE NOCASE ASC", [playlist, req.user.username], function(err, rows){ + var returnThis = []; + + for (var i = 0; i < rows.length; i++) { + + // var tempName = rows[i].filepath.split('/').slice(-1)[0]; + var tempName = fe.basename(rows[i].filepath); + var extension = getFileType(rows[i].filepath); + var filepath = fe.relative(req.user.musicDir, rows[i].filepath); + filepath = filepath.replace(/\\/g, '/'); + + returnThis.push({filepath: filepath, metadata:'' }); + } + + res.json(returnThis); + }); + }); + mstream.post('/playlist/delete', function(req, res){ + var playlistname = req.body.playlistname; + + // Handle a soft delete + if(req.body.hide && parseInt(req.body.hide) == true ){ + db.run("UPDATE mstream_playlists SET hide = 1 WHERE playlist_name = ? AND user = ?;", [playlistname, req.user.username], function(){ + res.json({success: true}); + }); + }else{ + // Delete playlist from DB + db.run("DELETE FROM mstream_playlists WHERE playlist_name = ? AND user = ?;", [playlistname, req.user.username], function(){ + res.json({success: true}); + }); + } + }); + + // TODO: Re-implment search + mstream.post('/db/search', function(req, res){ + res.json({error: 'search hdisabled for lokiJS'}); + }); + + mstream.get('/db/artists', function (req, res) { + var artists = {"artists":[]}; + var result = fileCollection.mapReduce(function(obj) { + return obj.artist; + }, function(arr) { + for(var i = 0; i < arr.length; i++) { + if(artists.artists.indexOf(arr[i]) === -1) { + console.log(arr[i]) + artists.artists.push(arr[i]); + } + } + return; + }); + + // artists.artists = result; + res.json(artists); + }); + + mstream.post('/db/artists-albums', function (req, res) { + var albums = {"albums":[]}; + + var results = fileCollection.find({ + '$and': [{ + 'user' : { '$eq' : req.user.username} + },{ + 'artist' : { '$eq' : req.body.artist} + }] + }); + + for(row of results){ + albums.albums.push({ + name: row.album, + album_art_file: row.albumArtFilename + }); + + res.json(albums); + } + }); + + mstream.get('/db/albums', function (req, res) { + var albums = {"albums":[]}; + + var result = fileCollection.mapReduce(function(obj) { + return {'name': obj.album, 'album_art_file': obj.albumArtFilename}; + }, function(arr) { + var ret = []; + var len = arr.length; + var store = []; + for(var i = 0; i < len; i++) { + console.log(arr[i]) + if(store.indexOf(arr[i].name) === -1) { + store.push(arr[i].name); + ret.push(arr[i]); + } + } + return ret; + }); + + console.log(result) + + albums.albums = result; + res.json(albums); + }); + + mstream.post('/db/album-songs', function (req, res) { + var results = fileCollection.find({ + '$and': [{ + 'user' : { '$eq' : req.user.username} + },{ + 'album' : { '$eq' : req.body.album} + }] + }); + var songs = []; + + for(row of results){ + var relativePath = fe.relative(req.user.musicDir, row.filepath); + relativePath = relativePath.replace(/\\/g, '/'); + + songs.push({ + "filepath": relativePath, + "metadata": { + "hash": row.hash, + "artist": row.artist, + "album": row.album, + "track": row.track, + "title": row.title, + "year": row.year, + "album-art": row.albumArtFilename, + "filename": fe.basename( row.filepath ) + } + }) + } + res.json(songs); + + }); + +} diff --git a/modules/db-write/database-default-loki.js b/modules/db-write/database-default-loki.js new file mode 100644 index 0000000..53c17e4 --- /dev/null +++ b/modules/db-write/database-default-loki.js @@ -0,0 +1,87 @@ +// These functions will take in JSON arrays of song data and then save that dat to the DB +const loki = require('lokijs'); +const filesdb = new loki('files.db'); +var fileCollection; + +// Add a collection to the database +// const fileCollection = filesdb.addCollection('files'); + +exports.setup = function(dbPath, callback){ + filesdb.loadDatabase({}, function(err) { + if (err) { + console.log("error : " + err); + } + else { + console.log("database loaded."); + } + + fileCollection = filesdb.getCollection("files"); + if (fileCollection === null) { + // first time run so add and configure collection with some arbitrary options + fileCollection = filesdb.addCollection("files"); + } + + callback() + }); +} + +exports.savedb = function(callback){ + filesdb.saveDatabase(function(err) { + if (err) { + console.log("error : " + err); + } + else { + console.log("database saved."); + } + callback() + }); +} + +exports.getUserFiles = function(thisUser, callback){ + var results = fileCollection.find({ user: thisUser.username }); + if(!results){ + results = []; + } + callback(results); +} + +/** + * @param arrayOfSongs + * @param username + * @return Promise + */ +exports.insertEntries = function(arrayOfSongs, username){ + return new Promise(function(resolve, reject) { + while(arrayOfSongs.length > 0) { + var song = arrayOfSongs.pop(); + console.log('BONG'); + + var doc = { + "title": song.title, + "artist": song.artist, + "year": song.year, + "artist": song.artist, + "album": song.album, + "filepath": song.filePath, + "format": song.format, + "track": song.track.no, + "disk": song.disk.no, + "filesize": song.filesize, + "modified": song.modified, + "created": song.created, + "hash": song.hash, + "albumArtFilename": song.albumArtFilename, + "user": username, + }; + fileCollection.insert(doc); + } + + resolve(); + }); +} + + +exports.deleteFile = function(path, user, callback){ + fileCollection.findAndRemove({'filePath': { '$eq' : path }}); + callback(); +} diff --git a/package.json b/package.json index 2d23dc3..2e51c3d 100644 --- a/package.json +++ b/package.json @@ -16,18 +16,17 @@ "author": "iros", "license": "GPL-3.0", "dependencies": { - "archiver": "^1.0.0", + "archiver": "^2.0.3", "body-parser": "^1.15.1", "commander": "^2.9.0", "express": "^4.13.4", - "internal-ip": "^1.2.0", - "jsonwebtoken": "^7.1.9", + "internal-ip": "^3.0.0", + "jsonwebtoken": "^8.1.0", "lokijs": "^1.4.3", "mkdirp": "^0.5.1", "music-metadata": "^0.8.4", "nat-upnp": "^1.1.0", "public-ip": "^2.0.1", - "sqlite3": "3.1.8", "uuid": "^3.0.1", "ws": "^1.1.1" },