diff --git a/modules/db-management/database-default-manager-old.js b/modules/db-management/database-default-manager-old.js new file mode 100644 index 0000000..42a5ac9 --- /dev/null +++ b/modules/db-management/database-default-manager-old.js @@ -0,0 +1,354 @@ + + +// This is designed to run as it's own process +// It takes in a json array +// { +// "username":"lol", +// "userDir":"/Users/psori/Desktop/Blockhead", +// "dbSettings":{ +// "type":"sqlite", +// "dbPath":"/Users/psori/Desktop/LATESTGREATEST.DB" +// } +// } + +const metadata = require('musicmetadata'); +const fs = require('fs'); +const fe = require('path'); +const crypto = require('crypto'); + + +try{ + var loadJson = JSON.parse(process.argv[process.argv.length-1], 'utf8'); + +}catch(error){ + console.log('Cannot parse JSON input'); + process.exit(); +} + + +// TODO: Check JSON for necessary info + + + + +// 2.0 + // Get all files from DB + // Hash file. If file and hash are found in array, then skip + // Seperate into new files and files that need to be updated + // Send these arrays to functions in database-default-X.js + + +var arrayOfSongs = []; // Holds songs for DB to process // TODO: Move out of global scope +var arrayOfScannedFiles = []; // Holds files for from recursive scan + + +// Pull in correct module +console.log(loadJson.dbSettings.type); +// TODO: Rename this var +const dbRead = require('../db-write/database-default-'+loadJson.dbSettings.type+'.js'); +if(loadJson.dbSettings.type == 'sqlite'){ + dbRead.setup(loadJson.dbSettings.dbPath); // TODO: Pass this in +} + + +// New way to start it +const parseFilesGenerator = rescanAllDirectories(loadJson.userDir); +parseFilesGenerator.next(); + + +function *rescanAllDirectories(directoryToScan){ + // Scan the directory for new, modified, and deleted files + var filesToProcess = yield rescanDirectory(directoryToScan); + + // Process all new files + if(filesToProcess.newFiles.length != 0){ + while(filesToProcess.newFiles.length > 0) { + yield parseFile(filesToProcess.newFiles.pop()); + } + // Finish inserting all new entries + yield insertEntries(50, true); + } + + + // process all updated files + while(filesToProcess.updatedFiles.length > 0) { + // Handle Editted songs + yield parseUpdatedSong(filesToProcess.updatedFiles.pop()); + yield insertEntries(50, true); + } + + // TODO: Process deleted files + while(filesToProcess.deletedFiles.length > 0) { + // Handle Editted songs + yield deleteFile(filesToProcess.deletedFiles.pop()); + } + + // Exit + process.exit(0); +} + +function rescanDirectory(dir){ + + // Get all files from DB + // TODO: Move This + dbRead.getUserFiles(loadJson, function(rows){ + + // Scan through files + var fileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; + recursiveScan(dir, fileTypesArray); + + var latestFileList = arrayOfScannedFiles; + arrayOfScannedFiles = []; + + + + var dbFileList = []; + var dbFileListTimestamp = []; + + for(var s of rows){ + console.log(s); + dbFileList.push(s.path); + + dbFileListTimestamp.push(s.path + '::' + s.file_modified_date); + } + + console.log(dbFileList); + console.log(dbFileListTimestamp); + + + + var arrayOfUpdatedSongsToProcess = []; + var arrayOfSongsToProcess = []; + var deletedFiles = []; // TODO: Global variable ??? + var checkForModifications = []; + + + + var latestFileListSet = new Set(latestFileList); + var dbFileListSet = new Set(dbFileList); + var dbFileListTimestampSet = new Set(dbFileListTimestamp); + + + // Get deleted files + dbFileList.filter(function(x) { + if( latestFileListSet.has(x) ){ + + }else{ + // It's deleted ( i think) + deletedFiles.push(x); + } + }); + + console.log('DELETED FILES'); + console.log(deletedFiles); + + + // Get new files + latestFileList.filter(function(x) { + + console.log(x); + + if(dbFileListSet.has(x)){ + checkForModifications.push(x + "::" + fs.statSync(x).mtime.getTime() ); + console.log('yes'); + + }else{ + // New files + arrayOfSongsToProcess.push(x); + console.log('no'); + + } + }); + + + console.log('NEW FILES'); + console.log(arrayOfSongsToProcess); + + // loop through checkForModifications + // Append timestamp to all path strings + // Compare dbFileList clone with path strings appended + checkForModifications.filter(function(x) { + if(dbFileListTimestampSet.has(x)){ + + }else{ + // File x has been updated + var filePath = x.split("::")[0]; + arrayOfUpdatedSongsToProcess.push(filePath); + } + }); + + console.log('POTENTIALLY UPDATED SONGS'); + console.log(arrayOfUpdatedSongsToProcess); + + + + // TODO: Handle deleted files + // We need to prompt users to see if they want to delete files on the server side + // We can store a default behaviour + + var returnArray = { + "newFiles":arrayOfSongsToProcess, + "updatedFiles":arrayOfUpdatedSongsToProcess, + "deletedFiles":deletedFiles + }; + parseFilesGenerator.next(returnArray); + }); + +} + +function parseUpdatedSong(filePath){ + // Check sha256 hash and confirm it has changed + // Update file status in DB accordingly + + + var fileStream = fs.createReadStream(filePath); + + var hash = crypto.createHash('sha256'); + hash.setEncoding('hex'); + + + fileStream.on('end', function () { + hash.end(); + + var hashIt = String(hash.read()); + + // compare hashes + //db.all("SELECT * FROM files WHERE path=? AND hash=?", [filePath, hashIt], function(err, rows){ + dbRead.getHashedEntry(hashIt, filePath, loadJson.username, function(rows){ + console.log(rows); + // No match found, file needs to be updated + if( !rows || rows.length === 0 ){ + // TODO: delete entry + dbRead.deleteFile(filePath, loadJson.username, function(){ + // Re-add entry + parseFile(filePath); + }); + + }else{ + parseFilesGenerator.next(); + + } + + }); + }); + + fileStream.pipe(hash); +} + + +function deleteFile(filepath){ + dbRead.deleteFile(filepath, loadJson.username, function(){ + // Re-add entry + parseFilesGenerator.next(); + }); +} + +function parseFile(thisSong){ + var filestat = fs.statSync(thisSong); + if(!filestat.isFile()){ + // TODO: Something is fucky, log it + parseFilesGenerator.next(); + return; + } + + // Stores all data that needs to be added to DB + var songInfo; + + // TODO: Hash the file here and add the hash to the DB + var hash = crypto.createHash('sha256'); + hash.setEncoding('hex'); + + var readableStream = fs.createReadStream(thisSong); + var parser = metadata(readableStream, function (err, thisMetadata) { + if(err){ + // TODO: Do something + } + console.log(songInfo); + console.log(filestat); + + songInfo = thisMetadata; + songInfo.filesize = filestat.size; + songInfo.created = filestat.birthtime.getTime(); + songInfo.modified = filestat.mtime.getTime(); + songInfo.filePath = thisSong; + songInfo.format = getFileType(thisSong); + + + + readableStream.on('end', function () { + hash.end(); + readableStream.close(); + + songInfo.hash = String(hash.read()); + + console.log('XXXXXXXXXXXXXXXxx'); + console.log(songInfo); + + arrayOfSongs.push(songInfo); + + // if there are more than 100 entries, or if it's the last song + if(arrayOfSongs.length > 99){ + // Insert entries into DB + insertEntries(99, false); + }else{ + // For the generator + parseFilesGenerator.next(); + } + + }); + + + }); + readableStream.pipe(hash); +} + +function getFileType(filename){ + return filename.split(".").pop(); +} + +function recursiveScan(dir, fileTypesArray){ + var files = fs.readdirSync( dir ); + + + // loop through files + for (var i=0; i < files.length; i++) { + // var filePath = dir + files[i]; + var filePath = fe.join(dir, files[i]); + // console.log(filePath); + var stat = fs.statSync(filePath); + + + if(stat.isDirectory()){ + recursiveScan(filePath, fileTypesArray); + }else{ + var extension = getFileType(files[i]); + + // Make sure this is in our list of allowed files + if (fileTypesArray.indexOf(extension) > -1 ) { + arrayOfScannedFiles.push(filePath); + } + } + } +} + + +function insertEntries(numberToInsert = 99, loopToEnd = false){ + var insertThese = []; + + while(insertThese.length != numberToInsert ){ + if(arrayOfSongs.length == 0){ + break; + } + insertThese.push(arrayOfSongs.pop()); + } + + dbRead.insertEntries(insertThese, loadJson.username, function(){ + // Recursivly run this function until all songs have been added + if(loopToEnd && arrayOfSongs.length != 0){ + insertEntries(numberToInsert, true); + }else{ + // For the generator + parseFilesGenerator.next(); + } + }); +} diff --git a/modules/db-management/database-default-manager.js b/modules/db-management/database-default-manager.js index 4bcf996..f5f9a48 100644 --- a/modules/db-management/database-default-manager.js +++ b/modules/db-management/database-default-manager.js @@ -1,7 +1,3 @@ -#!/usr/bin/env node -"use strict"; - - // This is designed to run as it's own process // It takes in a json array // { @@ -21,234 +17,126 @@ const crypto = require('crypto'); try{ var loadJson = JSON.parse(process.argv[process.argv.length-1], 'utf8'); - }catch(error){ console.log('Cannot parse JSON input'); process.exit(); } - -// TODO: Check JSON for nencessary info - - - - -// 2.0 - // Get all files from DB - // Hash file. If file and hash are found in array, then skip - // Seperate into new files and files that need to be updated - // Send these arrays to functions in database-default-X.js - - -var arrayOfSongs = []; // Holds songs for DB to process // TODO: Move out of global scope -var arrayOfScannedFiles = []; // Holds files for from recursive scan - - -// Pull in correct module -console.log(loadJson.dbSettings.type); -// TODO: Rename this var const dbRead = require('../db-write/database-default-'+loadJson.dbSettings.type+'.js'); if(loadJson.dbSettings.type == 'sqlite'){ - dbRead.setup(loadJson.dbSettings.dbPath); // TODO: Pass this in + dbRead.setup(loadJson.dbSettings.dbPath); } -// New way to start it const parseFilesGenerator = rescanAllDirectories(loadJson.userDir); parseFilesGenerator.next(); +var globalCurrentFileList = {}; + +var listOfFilesToParse = []; +var listOfFilesToDelete = []; + + function *rescanAllDirectories(directoryToScan){ // Scan the directory for new, modified, and deleted files - var filesToProcess = yield rescanDirectory(directoryToScan); + yield pullFromDB(); - // Process all new files - if(filesToProcess.newFiles.length != 0){ - while(filesToProcess.newFiles.length > 0) { - yield parseFile(filesToProcess.newFiles.pop()); - } - // Finish inserting all new entries - yield insertEntries(50, true); + // Loop through current files + recursiveScan(directoryToScan); + + console.log(listOfFilesToParse); + + + // for (var i=0; i < listOfFilesToDelete.length; i++) { + // yield deleteFile(listOfFilesToDelete[i]); + // } + + for (var i=0; i < listOfFilesToParse.length; i++) { + console.log(i); + yield parseFile(listOfFilesToParse[i]); } - - // process all updated files - while(filesToProcess.updatedFiles.length > 0) { - // Handle Editted songs - yield parseUpdatedSong(filesToProcess.updatedFiles.pop()); - yield insertEntries(50, true); - } - - // TODO: Process deleted files - while(filesToProcess.deletedFiles.length > 0) { - // Handle Editted songs - yield deleteFile(filesToProcess.deletedFiles.pop()); - } + // Anything left in globalCurrentFileList at this point has been deleted. Remove these from the database + // TODO: delete files // Exit process.exit(0); } -function rescanDirectory(dir){ - - // Get all files from DB - // TODO: Move This +function pullFromDB(){ dbRead.getUserFiles(loadJson, function(rows){ - // Scan through files - var fileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; - recursiveScan(dir, fileTypesArray); - - var latestFileList = arrayOfScannedFiles; - arrayOfScannedFiles = []; - - - - var dbFileList = []; - var dbFileListTimestamp = []; - for(var s of rows){ - console.log(s); - dbFileList.push(s.path); - - dbFileListTimestamp.push(s.path + '::' + s.file_modified_date); + globalCurrentFileList[s.path] = s; } - console.log(dbFileList); - console.log(dbFileListTimestamp); - - - - var arrayOfUpdatedSongsToProcess = []; - var arrayOfSongsToProcess = []; - var deletedFiles = []; // TODO: Global variable ??? - var checkForModifications = []; - - - - var latestFileListSet = new Set(latestFileList); - var dbFileListSet = new Set(dbFileList); - var dbFileListTimestampSet = new Set(dbFileListTimestamp); - - - // Get deleted files - dbFileList.filter(function(x) { - if( latestFileListSet.has(x) ){ - - }else{ - // It's deleted ( i think) - deletedFiles.push(x); - } - }); - - console.log('DELETED FILES'); - console.log(deletedFiles); - - - // Get new files - latestFileList.filter(function(x) { - - console.log(x); - - if(dbFileListSet.has(x)){ - checkForModifications.push(x + "::" + fs.statSync(x).mtime.getTime() ); - console.log('yes'); - - }else{ - // New files - arrayOfSongsToProcess.push(x); - console.log('no'); - - } - }); - - - console.log('NEW FILES'); - console.log(arrayOfSongsToProcess); - - // loop through checkForModifications - // Append timestamp to all path strings - // Compare dbFileList clone with path strings appended - checkForModifications.filter(function(x) { - if(dbFileListTimestampSet.has(x)){ - - }else{ - // File x has been updated - var filePath = x.split("::")[0]; - arrayOfUpdatedSongsToProcess.push(filePath); - } - }); - - console.log('POTENTIALLY UPDATED SONGS'); - console.log(arrayOfUpdatedSongsToProcess); - - - - // TODO: Handle deleted files - // We need to prompt users to see if they want to delete files on the server side - // We can store a default behaviour - - var returnArray = { - "newFiles":arrayOfSongsToProcess, - "updatedFiles":arrayOfUpdatedSongsToProcess, - "deletedFiles":deletedFiles - }; - parseFilesGenerator.next(returnArray); - }); - -} - -function parseUpdatedSong(filePath){ - // Check sha256 hash and confirm it has changed - // Update file status in DB accordingly - - - var fileStream = fs.createReadStream(filePath); - - var hash = crypto.createHash('sha256'); - hash.setEncoding('hex'); - - - fileStream.on('end', function () { - hash.end(); - - var hashIt = String(hash.read()); - - // compare hashes - //db.all("SELECT * FROM files WHERE path=? AND hash=?", [filePath, hashIt], function(err, rows){ - dbRead.getHashedEntry(hashIt, filePath, loadJson.username, function(rows){ - console.log(rows); - // No match found, file needs to be updated - if( !rows || rows.length === 0 ){ - // TODO: delete entry - dbRead.deleteFile(filePath, loadJson.username, function(){ - // Re-add entry - parseFile(filePath); - }); - - }else{ - parseFilesGenerator.next(); - - } - - }); - }); - - fileStream.pipe(hash); -} - - -function deleteFile(filepath){ - dbRead.deleteFile(filepath, loadJson.username, function(){ - // Re-add entry parseFilesGenerator.next(); }); } + + +function recursiveScan(dir, fileTypesArray){ + var files = fs.readdirSync( dir ); + + // loop through files + for (var i=0; i < files.length; i++) { + var filepath = fe.join(dir, files[i]); + try{ + var stat = fs.statSync(filepath); + }catch(error){ + // Bad file, ignore and continue + continue; + } + + if(stat.isDirectory()){ + recursiveScan(filepath); + }else{ + // Make sure this is in our list of allowed files + var extension = getFileType(files[i]); + var fileTypesArray = ["mp3", "flac", "wav", "ogg", "aac", "m4a"]; + + if (fileTypesArray.indexOf(extension) === -1 ) { + continue; + } + console.log(filepath); + + + // Check if in globalCurrentFileList + if (!(filepath in globalCurrentFileList)){ + // TODO: if Not parse new file, add it to DB, and continue + listOfFilesToParse.push(filepath); + // yield parseFile(filepath); + continue; + } + + // check the file_modified_date + if(stat.mtime.getTime() !== globalCurrentFileList[filepath].file_modified_date){ + listOfFilesToParse.push(filepath); + listOfFilesToDelete.push(filepath); + + + // TODO: If they are not the same, parse and update + // yield deleteFile(); + // yield parseFile(filepath); + } + + // Remove from globalCurrentFileList + delete globalCurrentFileList[filepath]; + } + } +} + + + function parseFile(thisSong){ + console.log(thisSong); var filestat = fs.statSync(thisSong); + console.log(filestat); if(!filestat.isFile()){ // TODO: Something is fucky, log it + console.log('BAD FILE'); parseFilesGenerator.next(); return; } @@ -256,17 +144,19 @@ function parseFile(thisSong){ // Stores all data that needs to be added to DB var songInfo; - // TODO: Hash the file here and add the hash to the DB - var hash = crypto.createHash('sha256'); - hash.setEncoding('hex'); + + + + + var readableStream = fs.createReadStream(thisSong); var parser = metadata(readableStream, function (err, thisMetadata) { + readableStream.close(); + if(err){ // TODO: Do something } - console.log(songInfo); - console.log(filestat); songInfo = thisMetadata; songInfo.filesize = filestat.size; @@ -276,81 +166,37 @@ function parseFile(thisSong){ songInfo.format = getFileType(thisSong); + // TODO: Hash the file here and add the hash to the DB + var hash = crypto.createHash('sha256'); + hash.setEncoding('hex'); + var readableStream2 = fs.createReadStream(thisSong); - readableStream.on('end', function () { + readableStream2.on('end', function () { hash.end(); - readableStream.close(); + readableStream2.close(); songInfo.hash = String(hash.read()); - console.log('XXXXXXXXXXXXXXXxx'); console.log(songInfo); - arrayOfSongs.push(songInfo); - - // if there are more than 100 entries, or if it's the last song - if(arrayOfSongs.length > 99){ - // Insert entries into DB - insertEntries(99, false); - }else{ - // For the generator + dbRead.insertEntries([songInfo], loadJson.username, function(){ parseFilesGenerator.next(); - } - + }); }); + readableStream2.pipe(hash); + }); - readableStream.pipe(hash); +} + +function deleteFile(filepath){ + dbRead.deleteFile(filepath, loadJson.username, function(){ + // Re-add entry + parseFilesGenerator.next(); + }); } function getFileType(filename){ return filename.split(".").pop(); } - -function recursiveScan(dir, fileTypesArray){ - var files = fs.readdirSync( dir ); - - - // loop through files - for (var i=0; i < files.length; i++) { - // var filePath = dir + files[i]; - var filePath = fe.join(dir, files[i]); - // console.log(filePath); - var stat = fs.statSync(filePath); - - - if(stat.isDirectory()){ - recursiveScan(filePath, fileTypesArray); - }else{ - var extension = getFileType(files[i]); - - // Make sure this is in our list of allowed files - if (fileTypesArray.indexOf(extension) > -1 ) { - arrayOfScannedFiles.push(filePath); - } - } - } -} - - -function insertEntries(numberToInsert = 99, loopToEnd = false){ - var insertThese = []; - - while(insertThese.length != numberToInsert ){ - if(arrayOfSongs.length == 0){ - break; - } - insertThese.push(arrayOfSongs.pop()); - } - - dbRead.insertEntries(insertThese, loadJson.username, function(){ - // Recursivly run this function until all songs have been added - if(loopToEnd && arrayOfSongs.length != 0){ - insertEntries(numberToInsert, true); - }else{ - // For the generator - parseFilesGenerator.next(); - } - }); -} diff --git a/modules/db-management/database-master.js b/modules/db-management/database-master.js index cf4ba37..950ccb7 100644 --- a/modules/db-management/database-master.js +++ b/modules/db-management/database-master.js @@ -2,7 +2,7 @@ exports.setup = function(mstream, program){ const child = require('child_process'); const fe = require('path'); - + // Load the public DB plugin @@ -149,7 +149,7 @@ exports.setup = function(mstream, program){ // Get number of files in DB mstreamReadPublicDB.getNumberOfFiles(req.user.username, function(numOfFiles){ - var returnOnject = { + var returnObject = { locked: false, totalFileCount: numOfFiles, dbType: 'default' @@ -162,12 +162,12 @@ exports.setup = function(mstream, program){ // Check for beets if(program.database_plugin.type === 'beets' ){ - returnOnject.dbType = 'beets-default'; + returnObject.dbType = 'beets-default'; }else if((req.user.privateDBOptions && req.user.privateDBOptions.privateDB === 'BEETS')){ - returnOnject.dbType = 'beets-default'; + returnObject.dbType = 'beets-default'; } - res.json(returnOnject); + res.json(returnObject); });