Moved most parts over to Loki. Still need to move the playlists

modules/db-management/database-default-manager.js

@@ -26,10 +26,7 @@ const crypto = require('crypto');
 
 // Setup DB layer
 // The DB functions are dcoupled from this so they can easily be swapped out
-const dbRead = require('../db-write/database-default-'+loadJson.dbSettings.type+'.js');
-if(loadJson.dbSettings.type == 'sqlite'){
-  dbRead.setup(loadJson.dbSettings.dbPath);
-}
+const dbRead = require('../db-write/database-default-loki.js');
 
 // Global Vars
 var globalCurrentFileList = {};  // Map of file paths to metadata
@@ -42,22 +39,29 @@ parseFilesGenerator.next();
 
 // Scan the directory for new, modified, and deleted files
 function *rescanAllDirectories(directoryToScan){
+  yield dbRead.setup(loadJson.dbSettings.dbPath, function(){
+    parseFilesGenerator.next();
+  });
   // Pull filelist from DB
-  yield pullFromDB();
+  pullFromDB();
   // Loop through current files and compare them to the files pulled from the DB
   recursiveScan(directoryToScan);
   // Delete Files
   for (var i=0; i < listOfFilesToDelete.length; i++) {
-    yield deleteFile(listOfFilesToDelete[i]);
+    deleteFile(listOfFilesToDelete[i]);
   }
   // Delete all remaining files
   for (var file in globalCurrentFileList) {
-    yield deleteFile(file);
+    deleteFile(file);
   }
   // Parse and add files to DB
   for (var i=0; i < listOfFilesToParse.length; i++) {
     yield parseFile(listOfFilesToParse[i]);
   }
+
+  yield dbRead.savedb(function(){
+    parseFilesGenerator.next();
+  })
   // Exit
   process.exit(0);
 }
@@ -66,10 +70,8 @@ function *rescanAllDirectories(directoryToScan){
 function pullFromDB(){
   dbRead.getUserFiles(loadJson, function(rows){
     for(var s of rows){
-      globalCurrentFileList[s.path] = s;
+      globalCurrentFileList[s.filepath] = s;
     }
-    // Go to next step
-    parseFilesGenerator.next();
   });
 }
 
@@ -105,7 +107,7 @@ function recursiveScan(dir, fileTypesArray){
       }
 
       // check the file_modified_date
-      if(stat.mtime.getTime() !== globalCurrentFileList[filepath].file_modified_date){
+      if(stat.mtime.getTime() !== globalCurrentFileList[filepath].modified){
         listOfFilesToParse.push(filepath);
         listOfFilesToDelete.push(filepath);
       }
@@ -119,7 +121,6 @@ function recursiveScan(dir, fileTypesArray){
 
 
 function parseFile(thisSong){
-  // console.log(thisSong);
   var filestat = fs.statSync(thisSong);
   if(!filestat.isFile()){
     // TODO: Something is fucky, log it
@@ -161,7 +162,6 @@ function calculateHash (thisSong, songInfo) {
     if (songInfo.picture && songInfo.picture[0]) {
       bufferString = songInfo.picture[0].data.toString('utf8');
       picFormat = songInfo.picture[0].format;
-      // console.log(songInfo.picture);
     } else if (false) { // TODO: Check if there is album art in base folder
 
     }
@@ -197,6 +197,7 @@ function calculateHash (thisSong, songInfo) {
 }
 
 function deleteFile(filepath){
+  console.log(filepath)
   dbRead.deleteFile(filepath, loadJson.username, function(){
     // Re-add entry
     parseFilesGenerator.next();

modules/db-management/database-master.js

@@ -4,7 +4,7 @@ exports.setup = function(mstream, program){
 
   // Load in API enndpoints
   // TODO: Change the name of this file
-  const mstreamReadPublicDB = require('../db-read/database-public-sqlite.js');
+  const mstreamReadPublicDB = require('../db-read/database-public-loki.js');
   mstreamReadPublicDB.setup(mstream, program.database_plugin);
 
   // Var that keeps track of DB scans going on
@@ -225,7 +225,7 @@ exports.setup = function(mstream, program){
         username: username,
         musicDir: program.users[username].musicDir,
       }, function(){
-        // TODO: Add generator and yield here
+        mstreamReadPublicDB.loadDB();
         bootScanGenerator.next();
       });
     }

modules/db-read/database-public-loki.js

@@ -0,0 +1,263 @@
+// TODO: This style looks up things synchronously from lokijs
+// ideally we would ru na loki server on a seperate thread so we don't block our server
+
+const fe = require('path')
+const crypto = require('crypto')
+
+// These functions will take in JSON arrays of song data and then save that dat to the DB
+const loki = require('lokijs')
+const filesdb = new loki('files.db')
+
+var fileCollection
+var playlistColection
+
+
+
+function loadDB(){
+  filesdb.loadDatabase({}, function(err) {
+    if (err) {
+      console.log("error : " + err);
+    }
+    else {
+      console.log("database loaded XXX");
+    }
+    // Add a collection to the database
+    fileCollection = filesdb.getCollection('files')
+    playlistColection = filesdb.getCollection('playlists')
+
+  });
+}
+
+// Load DB on boot
+loadDB();
+
+exports.loadDB = function(){
+  loadDB();
+}
+
+function getFileType(filename){
+  return filename.split(".").pop()
+}
+
+exports.getNumberOfFiles = function(username, callback){
+  var results = fileCollection.count({ 'user': username })
+  callback(results)
+}
+
+exports.setup = function (mstream, dbSettings){
+  // Metadata lookup
+  mstream.post('/db/metadata', function (req, res){
+    var relativePath = req.body.filepath;
+    var fullpath = fe.join(req.user.musicDir, relativePath);
+
+    var result = fileCollection.findOne({'filepath': fullpath});
+    res.json({
+      "filepath":relativePath,
+      "metadata":{
+        "artist":result.artist,
+        "hash": result.hash,
+        "album":result.album,
+        "track":result.track,
+        "title":result.title,
+        "year":result.year,
+        "album-art":row.albumArtFilename
+      }
+    });
+  });
+
+
+  // TODO: This needs to be tested to see if it works on extra large playlists (think thousands of entries)
+  // TODO: Ban saving playlists that are > 10,000 items long
+  mstream.post('/playlist/save', function (req, res){
+    var title = req.body.title;
+    var songs = req.body.songs;
+
+    // Check if this playlist already exists
+    db.all("SELECT id FROM mstream_playlists WHERE playlist_name = ? AND user = ?;", [title, req.user.username], function(err, rows) {
+      db.serialize(function() {
+        // We need to delete anys existing entries
+        if(rows && rows.length > 0){
+          db.run("DELETE FROM mstream_playlists WHERE playlist_name = ? AND user = ?;", [title, req.user.username]);
+        }
+
+        // Now we add the new entries
+        var sql2 = "insert into mstream_playlists (playlist_name, filepath, user) values ";
+        var sqlParser = [];
+
+        while(songs.length > 0) {
+          var song = songs.shift();
+
+          sql2 += "(?, ?, ?), ";
+          sqlParser.push(title);
+          sqlParser.push( fe.join(req.user.musicDir, song)  );
+          sqlParser.push( req.user.username );
+        }
+
+        sql2 = sql2.slice(0, -2);
+        sql2 += ";";
+
+        db.run(sql2, sqlParser, function(){
+          res.json({success: true});
+        });
+
+      });
+    });
+  });
+
+
+  // Attach API calls to functions
+  mstream.get('/playlist/getall', function (req, res){
+    // TODO: In V2 we need to change this to ignore hidden playlists
+    // TODO: db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE hide=0;", function(err, rows){
+    db.all("SELECT DISTINCT playlist_name FROM mstream_playlists WHERE user = ?", [req.user.username], function(err, rows){
+      var playlists = [];
+
+      // loop through files
+      for (var i = 0; i < rows.length; i++) {
+        if(rows[i].playlist_name){
+          playlists.push({name: rows[i].playlist_name});
+        }
+      }
+
+      res.json(playlists);
+    });
+  });
+
+  mstream.post('/playlist/load', function (req, res){
+    var playlist = req.body.playlistname;
+
+    db.all("SELECT * FROM mstream_playlists WHERE playlist_name = ? AND user = ? ORDER BY id  COLLATE NOCASE ASC", [playlist, req.user.username], function(err, rows){
+      var returnThis = [];
+
+      for (var i = 0; i < rows.length; i++) {
+
+        // var tempName = rows[i].filepath.split('/').slice(-1)[0];
+        var tempName = fe.basename(rows[i].filepath);
+        var extension = getFileType(rows[i].filepath);
+        var filepath = fe.relative(req.user.musicDir, rows[i].filepath);
+        filepath = filepath.replace(/\\/g, '/');
+
+        returnThis.push({filepath: filepath, metadata:'' });
+      }
+
+      res.json(returnThis);
+    });
+  });
+  mstream.post('/playlist/delete', function(req, res){
+    var playlistname = req.body.playlistname;
+
+    // Handle a soft delete
+    if(req.body.hide && parseInt(req.body.hide) == true ){
+      db.run("UPDATE mstream_playlists SET hide = 1 WHERE playlist_name = ? AND user = ?;", [playlistname, req.user.username], function(){
+        res.json({success: true});
+      });
+    }else{
+      // Delete playlist from DB
+      db.run("DELETE FROM mstream_playlists WHERE playlist_name = ? AND user = ?;", [playlistname, req.user.username], function(){
+        res.json({success: true});
+      });
+    }
+  });
+
+  // TODO: Re-implment search
+  mstream.post('/db/search', function(req, res){
+    res.json({error: 'search hdisabled  for lokiJS'});
+  });
+
+  mstream.get('/db/artists', function (req, res) {
+    var artists = {"artists":[]};
+    var result = fileCollection.mapReduce(function(obj) {
+    	return obj.artist;
+    }, function(arr) {
+      for(var i = 0; i <  arr.length; i++) {
+        if(artists.artists.indexOf(arr[i]) === -1) {
+          console.log(arr[i])
+          artists.artists.push(arr[i]);
+        }
+      }
+      return;
+    });
+
+    // artists.artists = result;
+    res.json(artists);
+  });
+
+  mstream.post('/db/artists-albums', function (req, res) {
+    var albums = {"albums":[]};
+
+    var results = fileCollection.find({
+      '$and': [{
+          'user' : { '$eq' :  req.user.username}
+        },{
+          'artist' :  { '$eq' :  req.body.artist}
+        }]
+    });
+
+    for(row of results){
+      albums.albums.push({
+        name: row.album,
+        album_art_file: row.albumArtFilename
+      });
+
+      res.json(albums);
+    }
+  });
+
+  mstream.get('/db/albums', function (req, res) {
+    var albums = {"albums":[]};
+
+    var result = fileCollection.mapReduce(function(obj) {
+    	return {'name': obj.album, 'album_art_file': obj.albumArtFilename};
+    }, function(arr) {
+    	var ret = [];
+      var len = arr.length;
+      var store = [];
+      for(var i = 0; i < len; i++) {
+        console.log(arr[i])
+        if(store.indexOf(arr[i].name) === -1) {
+          store.push(arr[i].name);
+          ret.push(arr[i]);
+        }
+      }
+      return ret;
+    });
+
+    console.log(result)
+
+    albums.albums = result;
+    res.json(albums);
+  });
+
+  mstream.post('/db/album-songs', function (req, res) {
+    var results = fileCollection.find({
+      '$and': [{
+          'user' : { '$eq' :  req.user.username}
+        },{
+          'album' :  { '$eq' :  req.body.album}
+        }]
+    });
+    var songs = [];
+
+    for(row of results){
+      var relativePath = fe.relative(req.user.musicDir, row.filepath);
+      relativePath = relativePath.replace(/\\/g, '/');
+
+      songs.push({
+        "filepath": relativePath,
+        "metadata": {
+          "hash": row.hash,
+          "artist": row.artist,
+          "album": row.album,
+          "track": row.track,
+          "title": row.title,
+          "year": row.year,
+          "album-art": row.albumArtFilename,
+          "filename":  fe.basename( row.filepath )
+        }
+      })
+    }
+    res.json(songs);
+
+  });
+
+}

modules/db-write/database-default-loki.js

@@ -0,0 +1,87 @@
+// These functions will take in JSON arrays of song data and then save that dat to the DB
+const loki = require('lokijs');
+const filesdb = new loki('files.db');
+var fileCollection;
+
+// Add a collection to the database
+// const fileCollection = filesdb.addCollection('files');
+
+exports.setup = function(dbPath, callback){
+  filesdb.loadDatabase({}, function(err) {
+    if (err) {
+      console.log("error : " + err);
+    }
+    else {
+      console.log("database loaded.");
+    }
+
+    fileCollection = filesdb.getCollection("files");
+    if (fileCollection === null) {
+      // first time run so add and configure collection with some arbitrary options
+      fileCollection = filesdb.addCollection("files");
+    }
+
+    callback()
+  });
+}
+
+exports.savedb = function(callback){
+  filesdb.saveDatabase(function(err) {
+    if (err) {
+      console.log("error : " + err);
+    }
+    else {
+      console.log("database saved.");
+    }
+    callback()
+  });
+}
+
+exports.getUserFiles = function(thisUser, callback){
+  var results = fileCollection.find({ user: thisUser.username });
+  if(!results){
+     results = [];
+  }
+  callback(results);
+}
+
+/**
+ * @param arrayOfSongs
+ * @param username
+ * @return Promise
+ */
+exports.insertEntries = function(arrayOfSongs, username){
+  return new Promise(function(resolve, reject) {
+    while(arrayOfSongs.length > 0) {
+      var song = arrayOfSongs.pop();
+      console.log('BONG');
+
+      var doc = {
+        "title": song.title,
+        "artist": song.artist,
+        "year": song.year,
+        "artist": song.artist,
+        "album": song.album,
+        "filepath": song.filePath,
+        "format": song.format,
+        "track": song.track.no,
+        "disk": song.disk.no,
+        "filesize": song.filesize,
+        "modified": song.modified,
+        "created": song.created,
+        "hash": song.hash,
+        "albumArtFilename": song.albumArtFilename,
+        "user": username,
+      };
+      fileCollection.insert(doc);
+    }
+
+    resolve();
+  });
+}
+
+
+exports.deleteFile = function(path, user, callback){
+  fileCollection.findAndRemove({'filePath': { '$eq' : path }});
+  callback();
+}

package.json

@@ -16,18 +16,17 @@
   "author": "iros",
   "license": "GPL-3.0",
   "dependencies": {
-    "archiver": "^1.0.0",
+    "archiver": "^2.0.3",
     "body-parser": "^1.15.1",
     "commander": "^2.9.0",
     "express": "^4.13.4",
-    "internal-ip": "^1.2.0",
-    "jsonwebtoken": "^7.1.9",
+    "internal-ip": "^3.0.0",
+    "jsonwebtoken": "^8.1.0",
     "lokijs": "^1.4.3",
     "mkdirp": "^0.5.1",
     "music-metadata": "^0.8.4",
     "nat-upnp": "^1.1.0",
     "public-ip": "^2.0.1",
-    "sqlite3": "3.1.8",
     "uuid": "^3.0.1",
     "ws": "^1.1.1"
   },