mirrors/mStream
3
0
Fork 0
mirror of https://github.com/IrosTheBeggar/mStream.git synced 2025-10-27 07:31:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

db restructure and optimizations

Browse Source
This commit is contained in:
IrosTheBeggar 2020-01-16 21:47:05 -05:00
parent 614e94cc12
commit 7041a09dfa
11 changed files with 88 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/API/db_status.md
View File

@ -18,11 +18,8 @@
```
{
locked: false,
totalFileCount: 150,
dbType: 'default'
totalFileCount: 150
}
```
`locked` will be true if a scan is in progress.
`dbType` will either be `default` or `beets`
`locked` will be true if a scan is in progress.

2
modules/config/config-inquirer.js
View File

@ -1027,7 +1027,7 @@ async function userLoop(loadJson) {
}
function editUser() {
console.log('NOT IMPLEMTEND');
console.log('NOT IMPLEMENTED');
return Promise.resolve();
}

48
modules/db-management/database-default-manager.js
View File

@ -3,7 +3,7 @@
// {
// "vpath":"metal",
// "directory":"/path/to/metal/music",
// "dbPath":"/path/to/LATESTGREATEST.DB",
// "dbPath":"/path/to/LATEST-GREATEST.DB",
// "pause": 500,
// "saveInterval": 1000,
// "skipImg":true
@ -64,11 +64,11 @@ function* scanDirectory(directoryToScan) {
}
// Delete all remaining files
for (var file in globalCurrentFileList) {
deleteFile(file);
deleteFile(fe.join(loadJson.directory, file));
}
// Parse and add files to DB
for (var i = 0; i < listOfFilesToParse.length; i++) {
yield parseFile(listOfFilesToParse[i]);
yield parseFile(fe.join(loadJson.directory, listOfFilesToParse[i]));
}
yield dbRead.savedb(() => {
@ -83,7 +83,7 @@ function* scanDirectory(directoryToScan) {
function pullFromDB() {
dbRead.getVPathFiles(loadJson.vpath, function (rows) {
for (var s of rows) {
globalCurrentFileList[s.filepath] = s;
globalCurrentFileList[s.filepath] = s.modified;
}
});
}
@ -115,15 +115,15 @@ function recursiveScan(dir) {
}
// Check if in globalCurrentFileList
if (!(filepath in globalCurrentFileList)) {
if (!(fe.relative(loadJson.directory, filepath) in globalCurrentFileList)) {
// if not parse new file, add it to DB, and continue
listOfFilesToParse.push(filepath);
listOfFilesToParse.push(fe.relative(loadJson.directory, filepath)); // use relative to remove extra data
continue;
}
// check the file_modified_date
if (stat.mtime.getTime() !== globalCurrentFileList[filepath].modified) {
listOfFilesToParse.push(filepath);
if (stat.mtime.getTime() !== globalCurrentFileList[fe.relative(loadJson.directory, filepath)]) {
listOfFilesToParse.push(fe.relative(loadJson.directory, filepath));
listOfFilesToDelete.push(filepath);
}
@ -133,11 +133,9 @@ function recursiveScan(dir) {
}
}
function parseFile(thisSong) {
var filestat = fs.statSync(thisSong);
if (!filestat.isFile()) {
var fileStat = fs.statSync(thisSong);
if (!fileStat.isFile()) {
console.error(`Warning: failed to parse file ${thisSong}: Unknown Error`);
parseFilesGenerator.next();
return;
@ -156,10 +154,8 @@ function parseFile(thisSong) {
console.error(`Warning: metadata parse error on ${thisSong}: ${err.message}`);
return {track: { no: null, of: null }, disk: { no: null, of: null }};
}).then(songInfo => {
songInfo.filesize = filestat.size;
songInfo.created = filestat.birthtime.getTime();
songInfo.modified = filestat.mtime.getTime();
songInfo.filePath = thisSong;
songInfo.modified = fileStat.mtime.getTime();
songInfo.filePath = fe.relative(loadJson.directory, thisSong);
songInfo.format = getFileType(thisSong);
// Calculate unique DB ID
return calculateHash(thisSong, songInfo);
@ -197,13 +193,13 @@ function calculateHash(thisSong, songInfo) {
}
// Album art has been pulled from directory already
else if (mapOfDirectoryAlbumArt.hasOwnProperty(fe.dirname(thisSong)) && mapOfDirectoryAlbumArt[fe.dirname(thisSong)] !== false) {
songInfo.albumArtFilename = mapOfDirectoryAlbumArt[fe.dirname(thisSong)];
songInfo.aaFile = mapOfDirectoryAlbumArt[fe.dirname(thisSong)];
}
// Directory has not been scanned for album art yet
else if (!mapOfDirectoryAlbumArt.hasOwnProperty(fe.dirname(thisSong))) {
var albumArt = checkDirectoryForAlbumArt(fe.dirname(thisSong));
if (albumArt) {
songInfo.albumArtFilename = albumArt;
songInfo.aaFile = albumArt;
}
}
@ -222,11 +218,11 @@ function calculateHash(thisSong, songInfo) {
if (bufferString) {
// Generate unique name based off hash of album art and metadata
const picHashString = crypto.createHash('md5').update(bufferString).digest('hex');
songInfo.albumArtFilename = picHashString + '.' + picFormat;
songInfo.aaFile = picHashString + '.' + picFormat;
// Check image-cache folder for filename and save if doesn't exist
if (!fs.existsSync(fe.join(loadJson.albumArtDirectory, songInfo.albumArtFilename))) {
if (!fs.existsSync(fe.join(loadJson.albumArtDirectory, songInfo.aaFile))) {
// Save file sync
fs.writeFileSync(fe.join(loadJson.albumArtDirectory, songInfo.albumArtFilename), songInfo.picture[0].data);
fs.writeFileSync(fe.join(loadJson.albumArtDirectory, songInfo.aaFile), songInfo.picture[0].data);
}
}
@ -297,16 +293,16 @@ function checkDirectoryForAlbumArt(directory) {
}
const picHashString = crypto.createHash('md5').update(imageBuffer.toString('utf8')).digest('hex');
const albumArtFilename = picHashString + '.' + picFormat;
const aaFile = picHashString + '.' + picFormat;
// Check image-cache folder for filename and save if doesn't exist
if (!fs.existsSync(fe.join(loadJson.albumArtDirectory, albumArtFilename))) {
if (!fs.existsSync(fe.join(loadJson.albumArtDirectory, aaFile))) {
// Save file sync
fs.writeFileSync(fe.join(loadJson.albumArtDirectory, albumArtFilename), imageBuffer);
fs.writeFileSync(fe.join(loadJson.albumArtDirectory, aaFile), imageBuffer);
}
mapOfDirectoryAlbumArt[directory] = albumArtFilename;
return albumArtFilename;
mapOfDirectoryAlbumArt[directory] = aaFile;
return aaFile;
}
function deleteFile(filepath) {

1
modules/db-management/database-master.js
View File

@ -80,7 +80,6 @@ exports.setup = function (mstream, program) {
mstreamReadPublicDB.getNumberOfFiles(req.user.vpaths, (numOfFiles) => {
res.json({
totalFileCount: numOfFiles,
dbType: 'default',
locked: isScanning
});
});

80
modules/db-read/database-public-loki.js
View File

@ -47,7 +47,7 @@ function getAllAlbumsForUser(user) {
for (let row of results) {
if (!store[row.album] && !(row.album === undefined || row.album === null)) {
albums.push({ name: row.album, album_art_file: row.albumArtFilename });
albums.push({ name: row.album, album_art_file: row.aaFile });
store[row.album] = true;
}
}
@ -128,24 +128,22 @@ exports.setup = function (mstream, program) {
// Metadata lookup
mstream.post('/db/metadata', (req, res) => {
const pathInfo = program.getVPathInfo(req.body.filepath);
if (pathInfo === false) {
res.status(500).json({ error: 'Could not find file' });
return;
}
const pathInfo = program.getVPathInfo(req.body.filepath, req.user);
if (!pathInfo) { return res.status(500).json({ error: 'Could not find file' }); }
if (!fileCollection) {
res.json({ "filepath": pathInfo.relativePath, "metadata": {} });
res.json({ "filepath": req.body.filepath, "metadata": {} });
return;
}
const result = fileCollection.findOne({ 'filepath': pathInfo.fullPath });
const result = fileCollection.findOne({ '$and': [{'filepath': pathInfo.relativePath}, {'vpath': pathInfo.vpath}] });
if (!result) {
res.json({ "filepath": pathInfo.relativePath, "metadata": {} });
res.json({ "filepath": req.body.filepath, "metadata": {} });
return;
}
res.json({
"filepath": pathInfo.relativePath,
"filepath": req.body.filepath,
"metadata": {
"artist": result.artist ? result.artist : null,
"hash": result.hash ? result.hash : null,
@ -153,7 +151,7 @@ exports.setup = function (mstream, program) {
"track": result.track ? result.track : null,
"title": result.title ? result.title : null,
"year": result.year ? result.year : null,
"album-art": result.albumArtFilename ? result.albumArtFilename : null,
"album-art": result.aaFile ? result.aaFile : null,
"rating": result.rating ? result.rating : null
}
});
@ -276,11 +274,13 @@ exports.setup = function (mstream, program) {
for (let row of results) {
// Look up metadata
const pathInfo = program.getVPathInfo(row.filepath);
const pathInfo = program.getVPathInfo(row.filepath, req.user);
if (!pathInfo) { return res.status(500).json({ error: 'Could not find file' }); }
let metadata = {};
if (fileCollection) {
const result = fileCollection.findOne({ 'filepath': pathInfo.fullPath });
const result = fileCollection.findOne({ '$and': [{'filepath': pathInfo.relativePath}, { 'vpath': pathInfo.vpath }] });
if (result) {
metadata = {
"artist": result.artist ? result.artist : null,
@ -289,7 +289,7 @@ exports.setup = function (mstream, program) {
"track": result.track ? result.track : null,
"title": result.title ? result.title : null,
"year": result.year ? result.year : null,
"album-art": result.albumArtFilename ? result.albumArtFilename : null,
"album-art": result.aaFile ? result.aaFile : null,
"rating": result.rating ? result.rating : null
};
}
@ -307,13 +307,11 @@ exports.setup = function (mstream, program) {
return res.status(500).json({ error: 'Playlist DB Not Initiated' });
}
const playlistname = req.body.playlistname;
// Delete existing playlist
playlistCollection.findAndRemove({
'$and': [
{ 'user': { '$eq': req.user.username }},
{ 'name': { '$eq': playlistname }}
{ 'name': { '$eq': req.body.playlistname }}
]
});
@ -355,7 +353,7 @@ exports.setup = function (mstream, program) {
if (!store[row.album]) {
albums.albums.push({
name: row.album,
album_art_file: row.albumArtFilename ? row.albumArtFilename : null
album_art_file: row.aaFile ? row.aaFile : null
});
store[row.album] = true;
}
@ -397,12 +395,8 @@ exports.setup = function (mstream, program) {
}).compoundsort(['track','filepath']).data();
for (let row of results) {
let relativePath = fe.relative(program.folders[row.vpath].root, row.filepath);
relativePath = fe.join(row.vpath, relativePath)
relativePath = relativePath.replace(/\\/g, '/');
songs.push({
"filepath": relativePath,
"filepath": fe.join(row.vpath, row.filepath).replace(/\\/g, '/'),
"metadata": {
"artist": row.artist ? row.artist : null,
"hash": row.hash ? row.hash : null,
@ -410,7 +404,7 @@ exports.setup = function (mstream, program) {
"track": row.track ? row.track : null,
"title": row.title ? row.title : null,
"year": row.year ? row.year : null,
"album-art": row.albumArtFilename ? row.albumArtFilename : null,
"album-art": row.aaFile ? row.aaFile : null,
"filename": fe.basename(row.filepath),
"rating": row.rating ? row.rating : null
}
@ -427,17 +421,14 @@ exports.setup = function (mstream, program) {
const rating = req.body.rating;
const pathInfo = program.getVPathInfo(req.body.filepath);
if (pathInfo === false) {
res.status(500).json({ error: 'Could not find file' });
return;
}
if (!pathInfo) { return res.status(500).json({ error: 'Could not find file' }); }
if (!fileCollection) {
res.status(500).json({ error: 'No DB' });
return;
}
const result = fileCollection.findOne({ 'filepath': pathInfo.fullPath });
const result = fileCollection.findOne({ '$and':[{ 'filepath': pathInfo.relativePath}, { 'vpath': pathInfo.vpath }] });
if (!result) {
res.status(500).json({ error: 'File not found in DB' });
return;
@ -511,19 +502,16 @@ exports.setup = function (mstream, program) {
randomSong = results[randomNumber];
}
let relativePath = fe.relative(program.folders[randomSong.vpath].root, randomSong.filepath);
relativePath = fe.join(randomSong.vpath, relativePath)
relativePath = relativePath.replace(/\\/g, '/');
returnThis.songs.push({
filepath: relativePath, metadata: {
"filepath": fe.join(randomSong.vpath, randomSong.filepath).replace(/\\/g, '/'),
"metadata": {
"artist": randomSong.artist ? randomSong.artist : null,
"hash": randomSong.hash ? randomSong.hash : null,
"album": randomSong.album ? randomSong.album : null,
"track": randomSong.track ? randomSong.track : null,
"title": randomSong.title ? randomSong.title : null,
"year": randomSong.year ? randomSong.year : null,
"album-art": randomSong.albumArtFilename ? randomSong.albumArtFilename : null,
"album-art": randomSong.aaFile ? randomSong.aaFile : null,
"rating": randomSong.rating ? randomSong.rating : null
}
});
@ -574,7 +562,7 @@ exports.setup = function (mstream, program) {
if (!store[row[resCol]]) {
returnThis.push({
name: row[resCol],
album_art_file: row.albumArtFilename ? row.albumArtFilename : null
album_art_file: row.aaFile ? row.aaFile : null
});
store[row[resCol]] = true;
}
@ -608,12 +596,8 @@ exports.setup = function (mstream, program) {
}).simplesort('rating', true).data();
for (let row of results) {
let relativePath = fe.relative(program.folders[row.vpath].root, row.filepath);
relativePath = fe.join(row.vpath, relativePath)
relativePath = relativePath.replace(/\\/g, '/');
songs.push({
"filepath": relativePath,
"filepath": fe.join(row.vpath, row.filepath).replace(/\\/g, '/'),
"metadata": {
"artist": row.artist ? row.artist : null,
"hash": row.hash ? row.hash : null,
@ -621,7 +605,7 @@ exports.setup = function (mstream, program) {
"track": row.track ? row.track : null,
"title": row.title ? row.title : null,
"year": row.year ? row.year : null,
"album-art": row.albumArtFilename ? row.albumArtFilename : null,
"album-art": row.aaFile ? row.aaFile : null,
"filename": fe.basename(row.filepath),
"rating": row.rating ? row.rating : null
}
@ -660,12 +644,8 @@ exports.setup = function (mstream, program) {
}).simplesort('ts', true).limit(limit).data();
for (let row of results) {
let relativePath = fe.relative(program.folders[row.vpath].root, row.filepath);
relativePath = fe.join(row.vpath, relativePath)
relativePath = relativePath.replace(/\\/g, '/');
songs.push({
"filepath": relativePath,
"filepath": fe.join(row.vpath, row.filepath).replace(/\\/g, '/'),
"metadata": {
"artist": row.artist ? row.artist : null,
"hash": row.hash ? row.hash : null,
@ -673,7 +653,7 @@ exports.setup = function (mstream, program) {
"track": row.track ? row.track : null,
"title": row.title ? row.title : null,
"year": row.year ? row.year : null,
"album-art": row.albumArtFilename ? row.albumArtFilename : null,
"album-art": row.aaFile ? row.aaFile : null,
"filename": fe.basename(row.filepath),
"rating": row.rating ? row.rating : null
}
@ -682,10 +662,6 @@ exports.setup = function (mstream, program) {
res.json(songs);
});
mstream.get('/db/recent/played', (req, res) => {
});
// Load DB on boot
loadDB();
}

6
modules/db-write/database-default-loki.js
View File

@ -71,14 +71,10 @@ exports.insertEntries = function (arrayOfSongs, vpath) {
"format": song.format,
"track": song.track.no ? song.track.no : null,
"disk": song.disk.no ? song.disk.no : null,
"filesize": song.filesize,
"modified": song.modified,
"created": song.created,
"hash": song.hash,
"albumArtFilename": song.albumArtFilename ? song.albumArtFilename : null,
"aaFile": song.aaFile ? song.aaFile : null,
"vpath": vpath,
"rating": 0,
"lastPlayed": 0,
"ts": Math.floor(Date.now() / 1000)
});

32
modules/defaults.js
View File

@ -3,7 +3,7 @@ const path = require('path');
const Joi = require('@hapi/joi');
exports.setup = function (config) {
config.filesDbName = 'files.loki-v1.db';
config.filesDbName = 'files.loki-v2.db';
const storageJoi = Joi.object({
albumArtDirectory: Joi.string().default(path.join(__dirname, '../image-cache')),
@ -106,40 +106,30 @@ exports.setup = function (config) {
}
// This is a convenience function. It gets the vPath from any url string
program.getVPathInfo = function (url) {
// TODO: Verify user has access to this vpath
program.getVPathInfo = function (url, user) {
// remove leading slashes
if (url.charAt(0) === '/') {
url = url.substr(1);
}
const fileArray = url.split('/');
const vpath = fileArray.shift();
// Get vpath from url
const vpath = url.split('/').shift();
// Verify user has access to this vpath
if (user && !user.vpaths.includes(vpath)) {
return false;
}
// Make sure the path exists
if (!program.folders[vpath]) {
return false;
}
const baseDir = program.folders[vpath].root;
let newPath = '';
for (const dir of fileArray) {
if (dir === '') {
continue;
}
newPath += dir + '/';
}
// TODO: There's gotta be a better way to construct the relative path
if (newPath.charAt(newPath.length - 1) === '/') {
newPath = newPath.slice(0, - 1);
}
return {
vpath: vpath,
basePath: baseDir,
relativePath: newPath,
fullPath: path.join(baseDir, newPath)
relativePath: path.relative(vpath, url),
fullPath: path.join(baseDir, path.relative(vpath, url))
};
}

7
modules/download.js
View File

@ -26,10 +26,9 @@ exports.setup = (mstream, program) => {
for (let i in fileArray) {
// TODO: Confirm each item in posted data is a real file
const pathInfo = program.getVPathInfo(fileArray[i]);
if (pathInfo === false) {
continue;
}
const pathInfo = program.getVPathInfo(fileArray[i], req.user);
if (!pathInfo) { continue; }
archive.file(pathInfo.fullPath, { name: fe.basename(fileArray[i]) });
}

9
modules/ffmpeg.js
View File

@ -39,11 +39,8 @@ exports.setup = (mstream, program) => {
ffmpeg.setFfprobePath(ffprobePath);
mstream.get("/transcode/*", (req, res) => {
const pathInfo = program.getVPathInfo(req.params[0]);
if (pathInfo === false) {
res.json({ "success": false });
return;
}
const pathInfo = program.getVPathInfo(req.params[0], req.user);
if (!pathInfo) { return res.json({ "success": false }); }
// Stream audio data
if (req.method === 'GET') {
@ -56,7 +53,7 @@ exports.setup = (mstream, program) => {
.audioCodec(codecMap[program.transcode.defaultCodec].codec)
.audioBitrate(program.transcode.defaultBitrate)
.on('end', () => {
// console.log('file has been converted succesfully');
// console.log('file has been converted successfully');
})
.on('error', err => {
winston.error('Transcoding Error!');

58
modules/file-explorer.js
View File

@ -20,13 +20,10 @@ const masterFileTypes = {
exports.setup = function(mstream, program) {
function getPathInfoOrThrow(req, pathString) {
const pathInfo = program.getVPathInfo(pathString);
const pathInfo = program.getVPathInfo(pathString, req.user);
if (pathInfo === false) {
throw {code: 500, json: { error: "Could not find file" }};
}
if (!req.user.vpaths.includes(pathInfo.vpath)) {
throw {code: 500, json: { error: "Access Denied" }};
}
return pathInfo;
}
@ -58,17 +55,8 @@ exports.setup = function(mstream, program) {
}
// Get full path
const pathInfo = program.getVPathInfo(req.body.directory);
if (pathInfo == false) {
res.status(500).json({ error: "Could not find file" });
return;
}
// Make sure the user has access to the given vpath and that the vpath exists
if (!req.user.vpaths.includes(pathInfo.vpath)) {
res.status(500).json({ error: "Access Denied" });
return;
}
const pathInfo = program.getVPathInfo(req.body.directory, req.user);
if (!pathInfo) { return res.status(500).json({ error: "Could not find file" }); }
// Make sure it's a directory
if (!fs.statSync(pathInfo.fullPath).isDirectory()) {
@ -121,12 +109,10 @@ exports.setup = function(mstream, program) {
if (!req.headers['data-location']) {
return res.status(500).json({ error: 'No Location Provided' });
}
const pathInfo = program.getVPathInfo(req.headers['data-location']);
if (!pathInfo.fullPath) {
return res.status(500).json({ error: 'Location could not be parsed' });
}
const pathInfo = program.getVPathInfo(req.headers['data-location'], req.user);
if (!pathInfo) { return res.status(500).json({ error: 'Location could not be parsed' }); }
// TODO: Check if path exits, if not make the path
// run make directory
try {
mkdirp.sync(pathInfo.fullPath);
} catch (err) {
@ -191,18 +177,8 @@ exports.setup = function(mstream, program) {
return res.json({ path: "/", contents: directories });
}
const directory = req.body.dir;
const pathInfo = program.getVPathInfo(directory);
if (pathInfo == false) {
res.status(500).json({ error: "Could not find file" });
return;
}
// Make sure the user has access to the given vpath and that the vpath exists
if (!req.user.vpaths.includes(pathInfo.vpath)) {
res.status(500).json({ error: "Access Denied" });
return;
}
const pathInfo = program.getVPathInfo(req.body.dir, req.user);
if (!pathInfo) { return res.status(500).json({ error: "Could not find file" }); }
// Make sure it's a directory
if (!fs.statSync(pathInfo.fullPath).isDirectory()) {
@ -249,7 +225,7 @@ exports.setup = function(mstream, program) {
});
// Format directory string for return value
let returnDirectory = directory.replace(/\\/g, "/");
let returnDirectory = req.body.dir.replace(/\\/g, "/");
if (returnDirectory.slice(-1) !== "/") {
returnDirectory += "/";
}
@ -259,22 +235,12 @@ exports.setup = function(mstream, program) {
});
mstream.post('/files/recursive-scan', function(req, res){
if(!req.body.dir) {
if (!req.body.dir) {
return res.status(422).json({ error: "Missing Directory" });
}
const directory = req.body.dir;
const pathInfo = program.getVPathInfo(directory);
if (pathInfo == false) {
res.status(500).json({ error: "Could not find file" });
return;
}
// Make sure the user has access to the given vpath and that the vpath exists
if (!req.user.vpaths.includes(pathInfo.vpath)) {
res.status(500).json({ error: "Access Denied" });
return;
}
const pathInfo = program.getVPathInfo(req.body.dir, req.user);
if (!pathInfo) { return res.status(500).json({ error: "Could not parse directory" }); }
// Make sure it's a directory
if (!fs.statSync(pathInfo.fullPath).isDirectory()) {

8
public/js/mstream.js
View File

@ -539,7 +539,7 @@ $(document).ready(function () {
}
// function that will recieve JSON array of a directory listing. It will then make a list of the directory and tack on classes for functionality
// function that will receive JSON array of a directory listing. It will then make a list of the directory and tack on classes for functionality
function printdir(response, previousState) {
currentBrowsingList = response.contents;
@ -1379,11 +1379,9 @@ $(document).ready(function () {
};
$('#filelist').on('submit', '#db-search', function (e) {
console.log('LOL');
$('#search-results').html('');
$('#search-results').append('<div class="loading-screen"><svg class="spinner" width="65px" height="65px" viewBox="0 0 66 66" xmlns="http://www.w3.org/2000/svg"><circle class="spinner-path" fill="none" stroke-width="6" stroke-linecap="round" cx="33" cy="33" r="30"></circle></svg></div>');
// Send AJAX Request
MSTREAMAPI.search($('#search-term').val(), function(res, error) {
if (error !== false) {
@ -1400,6 +1398,10 @@ $(document).ready(function () {
});
});
if (searchList.length < 2) {
searchList.push('<h5>No Results Found</h5>');
}
$('#search-results').html(searchList);
});
});