From e0dae61b9fc27d1b2a8dec17e56e49017bb052fd Mon Sep 17 00:00:00 2001 From: Martin Raiber Date: Sun, 9 Feb 2025 23:41:24 +0100 Subject: [PATCH] Add static file serving --- ApiHandler.cpp | 141 +++++++++++++++++++++++++++- ApiHandler.h | 33 +++++-- CMakeLists.txt | 2 + StaticHandler.cpp | 215 +++++++++++++++++++++++++++++++++++++++++++ StaticHandler.h | 85 +++++++++++++++++ build.sh | 23 +++++ main.cpp | 49 +++++++++- s3handler.cpp | 102 -------------------- s3handler.h | 3 - www/vite.config.ts | 13 ++- wwwgen/www_files.cpp | 18 ++++ wwwgen/www_files.h | 27 ++++++ wwwgen/wwwgen.py | 110 ++++++++++++++++++++++ 13 files changed, 700 insertions(+), 121 deletions(-) create mode 100644 StaticHandler.cpp create mode 100644 StaticHandler.h create mode 100755 build.sh create mode 100644 wwwgen/www_files.cpp create mode 100644 wwwgen/www_files.h create mode 100644 wwwgen/wwwgen.py diff --git a/ApiHandler.cpp b/ApiHandler.cpp index 964e1b8..faf58b6 100644 --- a/ApiHandler.cpp +++ b/ApiHandler.cpp @@ -27,6 +27,10 @@ #include "s3handler.h" #include "SingleFileStorage.h" #include "folly/ScopeGuard.h" +#include +#include "utils.h" + +using namespace proxygen; DEFINE_string(init_root_password, "", "Initial password of root account"); DEFINE_string(init_root_access_key, "", "Initial name of access key of root account. Default: root"); @@ -124,13 +128,79 @@ void ApiHandler::init() } } -ApiHandler::ApiHandler(const std::string_view func, const std::string_view cookieSes, S3Handler& s3handler) - : func(func), cookieSes(cookieSes), s3handler(s3handler) +ApiHandler::ApiHandler(SingleFileStorage &sfs) + : sfs(sfs), self(this) { +} + +void ApiHandler::onRequest(std::unique_ptr headers) noexcept +{ + if (headers->getMethod() != HTTPMethod::POST) + { + ResponseBuilder(downstream_) + .status(400, "Bad method") + .body("Only POST is supported") + .sendWithEOM(); + return; + } + + const std::string_view path(headers->getPathAsStringPiece()); + if(path.empty()) + return; + + const auto bucketEnd = path.find_first_of('/', 1); + if(bucketEnd == std::string::npos) + return; + + func = path.substr(bucketEnd+1); + + std::string cl = headers->getHeaders().getSingleOrEmpty( + proxygen::HTTP_HEADER_CONTENT_LENGTH); + if (cl.empty()) + { + ResponseBuilder(downstream_) + .status(500, "Internal error") + .body("Content-Length header not set") + .sendWithEOM(); + finished = true; + return; + } + if(func!="adduser" && func!="login" && func!="list") - throw FunctionNotFoundError(fmt::format("Api function \"{}\" not found", func)); + { + ResponseBuilder(downstream_) + .status(404, "Not found") + .body("Function not found") + .sendWithEOM(); + finished = true; + return; + } + + putRemaining = std::atoll(cl.c_str()); + + cookieSes = headers->getCookie("ses"); +} + +void ApiHandler::onBody(std::unique_ptr body) noexcept +{ + auto evb = folly::EventBaseManager::get()->getEventBase(); + + const auto bodyBytes = body->length(); + + doneBytes += bodyBytes; + putRemaining -= bodyBytes; + + if(!onBody(body->data(), body->length())) + { + ResponseBuilder(downstream_) + .status(500, "Internal error") + .body("Write ext error") + .sendWithEOM(); + finished = true; + return; + } } bool ApiHandler::onBody(const uint8_t* data, size_t dataSize) @@ -139,6 +209,70 @@ bool ApiHandler::onBody(const uint8_t* data, size_t dataSize) return true; } +void ApiHandler::onEOM() noexcept +{ + auto evb = folly::EventBaseManager::get()->getEventBase(); + + if(finished) + return; + + if(putRemaining!=0) + { + ResponseBuilder(downstream_) + .status(500, "Internal error") + .body("Expecting more data") + .sendWithEOM(); + finished = true; + return; + } + + folly::getGlobalCPUExecutor()->add( + [this, evb]() + { + const auto response = runRequest(); + + evb->runInEventBaseThread([this, response]() + { + auto statusMesg = response.code == 200 ? "OK" : "Internal error"; + + ResponseBuilder respBuilder(downstream_); + respBuilder.status(response.code, statusMesg); + respBuilder.body(response.contentType ? response.body : escapeXML(response.body)); + + if(response.setCookie) + { + respBuilder.header(HTTPHeaderCode::HTTP_HEADER_SET_COOKIE, fmt::format("ses={}; SameSite=Strict; HttpOnly", *response.setCookie)); + } + + if(response.contentType) + { + respBuilder.header(HTTPHeaderCode::HTTP_HEADER_CONTENT_TYPE, *response.contentType); + } + + respBuilder.sendWithEOM(); + + finished = true; }); + } + ); +} + +void ApiHandler::onUpgrade(UpgradeProtocol /*protocol*/) noexcept +{ + // handler doesn't support upgrades +} + +void ApiHandler::requestComplete() noexcept +{ + finished = true; + self.reset(); +} + +void ApiHandler::onError(proxygen::ProxygenError) noexcept +{ + finished = true; + self.reset(); +} + ApiHandler::ApiResponse ApiHandler::runRequest() { std::optional resp; @@ -259,7 +393,6 @@ Api::ListResp ApiHandler::list(const Api::ListParams& params, const ApiSessionSt if(!bucketIdOpt) throw ApiError(Api::Herror::bucketNotFound); const auto bucketId = *bucketIdOpt; - auto& sfs = s3handler.sfs; const auto iterStartVal = make_key({.key = params.continuationToken ? *params.continuationToken : prefix, .version=std::numeric_limits::max(), .bucketId = bucketId}); SingleFileStorage::IterData iterData = {}; diff --git a/ApiHandler.h b/ApiHandler.h index 84fc029..d62904c 100644 --- a/ApiHandler.h +++ b/ApiHandler.h @@ -13,23 +13,30 @@ #include "apigen/ListResp.hpp" #include "apigen/ListParams.hpp" #include "Session.h" +#include -class S3Handler; -class FunctionNotFoundError : std::runtime_error +class SingleFileStorage; + +class ApiHandler : public proxygen::RequestHandler { public: - FunctionNotFoundError(const std::string& msg) - : std::runtime_error(msg) {} -}; + ApiHandler(SingleFileStorage &sfs); -class ApiHandler -{ -public: - ApiHandler(const std::string_view func, const std::string_view cookieSes, S3Handler& s3handler); + void onRequest(std::unique_ptr headers) noexcept override; bool onBody(const uint8_t* data, size_t dataSize); + void onBody(std::unique_ptr body) noexcept override; + + void onEOM() noexcept override; + + void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override; + + void requestComplete() noexcept override; + + void onError(proxygen::ProxygenError err) noexcept override; + struct ApiResponse { int code; @@ -52,9 +59,15 @@ private: std::string body; std::string cookieSes; - S3Handler& s3handler; + SingleFileStorage &sfs; static thread_local DbDao dao; + + int64_t putRemaining = -1; + int64_t doneBytes = 0; + bool finished = false; + + std::unique_ptr self; }; class ApiError : std::runtime_error diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f71946..d6c84b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,6 +81,8 @@ add_executable(hs5 ApiHandler.cpp Session.cpp cmd.cpp + StaticHandler.cpp + wwwgen/www_files.cpp ${SCHEMA_SOURCES}) # Include the generated config.h diff --git a/StaticHandler.cpp b/StaticHandler.cpp new file mode 100644 index 0000000..b2989d6 --- /dev/null +++ b/StaticHandler.cpp @@ -0,0 +1,215 @@ +#include "StaticHandler.h" +#include "wwwgen/www_files.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace proxygen; + +void StaticHandler::onRequest(std::unique_ptr headers) noexcept +{ + _error = false; + if (headers->getMethod() != HTTPMethod::GET) + { + ResponseBuilder(downstream_) + .status(400, "Bad method") + .body("Only GET is supported") + .sendWithEOM(); + return; + } + + auto compressionOptions = CompressionFilterUtils::FactoryOptions(); + compressionOptions.enableZstd = false; + const auto compression = CompressionFilterUtils::getFilterParams(*headers, compressionOptions); + + if (!compression || compression->headerEncoding != "gzip") + { + _decompressor = std::make_unique(CompressionType::GZIP); + } + + const auto path = !headers->getPathAsStringPiece().empty() ? headers->getPathAsStringPiece().subpiece(1) : "index.html"; + const auto fpath = (path == "/" || path.empty()) ? "index.html" : path; + + const auto itFile = www_files.find(fpath); + + if (itFile == www_files.end()) + { + XLOGF(DBG0, "File not found: {}", fpath); + ResponseBuilder(downstream_) + .status(404, "Not found") + .body("File not found") + .sendWithEOM(); + return; + } + + const auto& etag = itFile->second.etag; + + const auto ifNoneMatch = headers->getHeaders().getSingleOrEmpty(HTTP_HEADER_IF_NONE_MATCH); + + if(ifNoneMatch == etag) + { + ResponseBuilder(downstream_) + .status(304, "Not modified") + .sendWithEOM(); + return; + } + + _file.set(std::string_view(reinterpret_cast(itFile->second.data), itFile->second.len)); + + ResponseBuilder resp(downstream_); + resp.status(200, "Ok") + .header(HTTP_HEADER_ETAG, etag) + .header(HTTP_HEADER_CONTENT_TYPE, itFile->second.contentType); + + if(itFile->second.immutable) + resp.header(HTTP_HEADER_CACHE_CONTROL, "Cache-Control: max-age=31536000, immutable"); + + if(!_decompressor) + { + resp.header(HTTP_HEADER_CONTENT_ENCODING, "gzip"); + resp.header(HTTP_HEADER_CONTENT_LENGTH, std::to_string(itFile->second.len)); + } + else + { + resp.header(HTTP_HEADER_CONTENT_LENGTH, std::to_string(itFile->second.uncompLen)); + } + + resp.send(); + + XLOGF(DBG0, "Sending file: {}", fpath); + + _readFileScheduled = true; + + folly::getGlobalCPUExecutor()->add( + [this, evb = folly::EventBaseManager::get()->getEventBase()]() + { + readFile(evb); + }); +} + +void StaticHandler::readFile(folly::EventBase *evb) +{ + folly::IOBufQueue buf; + while (!_file.done() && !_paused) + { + auto data = buf.preallocate(4000, 4000); + const auto rc = _file.read(reinterpret_cast(data.first), data.second); + if (rc == 0) + { + XLOGF(DBG0, "Read EOF"); + _file.setDone(); + evb->runInEventBaseThread([this] + { + if (!_error) { + ResponseBuilder(downstream_).sendWithEOM(); + } + }); + break; + } + else + { + buf.postallocate(rc); + XLOGF(DBG0, "Read {} bytes", rc); + if (_decompressor) + { + auto body = buf.move(); + auto decompBuf = _decompressor->decompress(body.get()); + XLOGF(DBG0, "Decompressed {} bytes", decompBuf->length()); + evb->runInEventBaseThread([this, body = std::move(decompBuf)]() mutable + { + if (!_error) { + ResponseBuilder(downstream_).body(std::move(body)).send(); + } + }); + } + else + { + evb->runInEventBaseThread([this, body = buf.move()]() mutable + { + if (!_error) { + ResponseBuilder(downstream_).body(std::move(body)).send(); + } + }); + } + } + } + + // Notify the request thread that we terminated the readFile loop + evb->runInEventBaseThread([this] + { + _readFileScheduled = false; + if (!checkForCompletion() && !_paused) { + VLOG(4) << "Resuming deferred readFile"; + onEgressResumed(); + } }); +} + +void StaticHandler::onEgressPaused() noexcept +{ + // This will terminate readFile soon + VLOG(4) << "StaticHandler paused"; + _paused = true; +} + +void StaticHandler::onEgressResumed() noexcept +{ + VLOG(4) << "StaticHandler resumed"; + _paused = false; + // If readFileScheduled_, it will reschedule itself + if (!_readFileScheduled && !_file.done()) + { + _readFileScheduled = true; + folly::getGlobalCPUExecutor()->add( + [this, evb = folly::EventBaseManager::get()->getEventBase()]() + { + readFile(evb); + }); + } + else + { + VLOG(4) << "Deferred scheduling readFile"; + } +} + +void StaticHandler::onBody(std::unique_ptr /*body*/) noexcept +{ +} + +void StaticHandler::onEOM() noexcept +{ +} + +void StaticHandler::onUpgrade(UpgradeProtocol /*protocol*/) noexcept +{ +} + +void StaticHandler::requestComplete() noexcept +{ + _finished = true; + _paused = true; + checkForCompletion(); +} + +void StaticHandler::onError(ProxygenError /*err*/) noexcept +{ + _error = true; + _finished = true; + _paused = true; + checkForCompletion(); +} + +bool StaticHandler::checkForCompletion() +{ + if (_finished && !_readFileScheduled) + { + VLOG(4) << "deleting StaticHandler"; + _self.reset(); + return true; + } + return false; +} \ No newline at end of file diff --git a/StaticHandler.h b/StaticHandler.h new file mode 100644 index 0000000..e2b055c --- /dev/null +++ b/StaticHandler.h @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace proxygen +{ + class ZlibStreamDecompressor; +} + +class MemFile +{ +public: + MemFile(const std::string_view data) + : _data(data) + {} + + void set(const std::string_view data) + { + _data = data; + } + + int read(char* buf, size_t bufSize) + { + if(_data.empty()) + return 0; + + const auto toRead = std::min(bufSize, _data.size()); + memcpy(buf, _data.data(), toRead); + _data.remove_prefix(toRead); + return toRead; + } + + bool done() const + { + return _done; + } + + void setDone() + { + _done = true; + } + +private: + std::string_view _data; + bool _done{false}; +}; + + +class StaticHandler : public proxygen::RequestHandler { + public: + StaticHandler() : _self(this) {} + + void onRequest(std::unique_ptr headers) noexcept override; + + void onBody(std::unique_ptr body) noexcept override; + + void onEOM() noexcept override; + + void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override; + + void requestComplete() noexcept override; + + void onError(proxygen::ProxygenError err) noexcept override; + + void onEgressPaused() noexcept override; + + void onEgressResumed() noexcept override; + + private: + void readFile(folly::EventBase* evb); + bool checkForCompletion(); + + MemFile _file{std::string_view()}; + bool _readFileScheduled{false}; + std::atomic _paused{false}; + bool _finished{false}; + std::atomic _error{false}; + std::unique_ptr _self; + std::unique_ptr _decompressor; +}; diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..0b5f261 --- /dev/null +++ b/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -ex + +CDIR="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +cd "$CDIR" + +cd www + +pnpm install --frozen-lockfile + +pnpm run build + +cd ../wwwgen + +python wwwgen.py ../www/dist not-empty + +cmake --preset ninja-multi-vcpkg + +cmake --build --preset ninja-vcpkg-release + + + diff --git a/main.cpp b/main.cpp index 5480ac6..82c1476 100644 --- a/main.cpp +++ b/main.cpp @@ -29,6 +29,7 @@ #include "ApiHandler.h" #include "Buckets.h" #include "config.h" +#include "StaticHandler.h" using namespace std::chrono_literals; @@ -67,7 +68,53 @@ class S3HandlerFactory : public proxygen::RequestHandlerFactory { void onServerStop() noexcept override { } - proxygen::RequestHandler* onRequest(proxygen::RequestHandler*, proxygen::HTTPMessage*) noexcept override { + bool isApiCall(const std::string_view path) + { + if(path.empty()) + return false; + + const auto bucketEnd = path.find_first_of('/', 1); + if(bucketEnd == std::string::npos) + return false; + const auto bucketName = path.substr(1, bucketEnd); + + if(bucketName!="api-v1-b64be512-4b03-4028-a589-13931942e205/") + return false; + + return true; + } + + bool isStaticFile(const std::string_view path) + { + if(path.empty()) + return true; + + if(path.size() == 1 && path[0]=='/') + return true; + + const auto bucketEnd = path.find_first_of('/', 1); + if(bucketEnd == std::string::npos) + return true; + + const auto bucketName = path.substr(1, bucketEnd); + + if(bucketName=="admin-b64be5124b034028a58913931942e205/") + return true; + + return false; + } + + proxygen::RequestHandler* onRequest(proxygen::RequestHandler*, proxygen::HTTPMessage* message) noexcept override + { + const std::string_view path(message->getPathAsStringPiece()); + + if(message->getMethod() == proxygen::HTTPMethod::POST && + isApiCall(path)) + return new ApiHandler(sfs); + if(message->getMethod() == proxygen::HTTPMethod::GET && + isStaticFile(path)) + return new StaticHandler(); + return new S3Handler(sfs, FLAGS_server_url, FLAGS_with_bucket_versioning); } }; diff --git a/s3handler.cpp b/s3handler.cpp index 32644b1..4b324f7 100644 --- a/s3handler.cpp +++ b/s3handler.cpp @@ -1055,9 +1055,6 @@ void S3Handler::onRequest(std::unique_ptr headers) noexcept } else if(headers->getMethod() == HTTPMethod::POST) { - if(handleApiCall(*headers)) - return; - if(!setKeyInfoFromPath(headers->getPathAsStringPiece())) return; @@ -1146,41 +1143,6 @@ void S3Handler::onRequest(std::unique_ptr headers) noexcept } } -bool S3Handler::handleApiCall(proxygen::HTTPMessage& headers) -{ - const std::string_view path(headers.getPathAsStringPiece()); - if(path.empty()) - return false; - - const auto bucketEnd = path.find_first_of('/', 1); - if(bucketEnd == std::string::npos) - return false; - - const auto bucketName = path.substr(1, bucketEnd); - - if(bucketName!="api-v1-b64be512-4b03-4028-a589-13931942e205/") - return false; - - const auto keyStr = path.substr(bucketEnd+1); - - std::string cl = headers.getHeaders().getSingleOrEmpty( - proxygen::HTTP_HEADER_CONTENT_LENGTH); - if (cl.empty()) - { - ResponseBuilder(downstream_) - .status(500, "Internal error") - .body("Content-Length header not set") - .sendWithEOM(); - return true; - } - - put_remaining = std::atoll(cl.c_str()); - - apiHandler = std::make_unique(keyStr, headers.getCookie("ses"), *this); - - return true; -} - bool S3Handler::setKeyInfoFromPath(const std::string_view path) { if(path.empty()) @@ -2714,24 +2676,6 @@ void S3Handler::onBody(std::unique_ptr body) noexcept size_t body_bytes = body->length(); - if(apiHandler) - { - done_bytes += body_bytes; - put_remaining.fetch_sub(body->length(), std::memory_order_release); - - if(!apiHandler->onBody(body->data(), body->length())) - { - ResponseBuilder(self->downstream_) - .status(500, "Internal error") - .body("Write ext error") - .sendWithEOM(); - finished_ = true; - return; - } - - return; - } - if(request_type == RequestType::CompleteMultipartUpload) { if(payloadHash) @@ -2970,52 +2914,6 @@ void S3Handler::onEOM() noexcept { auto evb = folly::EventBaseManager::get()->getEventBase(); - if(apiHandler) - { - if(finished_) - return; - - if(put_remaining!=0) - { - ResponseBuilder(self->downstream_) - .status(500, "Internal error") - .body("Expecting more data") - .sendWithEOM(); - finished_ = true; - return; - } - - folly::getGlobalCPUExecutor()->add( - [self = this->self, evb]() - { - const auto response = self->apiHandler->runRequest(); - - evb->runInEventBaseThread([self = self, response]() - { - auto statusMesg = response.code == 200 ? "OK" : "Internal error"; - - ResponseBuilder respBuilder(self->downstream_); - respBuilder.status(response.code, statusMesg); - respBuilder.body(response.contentType ? response.body : escapeXML(response.body)); - - if(response.setCookie) - { - respBuilder.header(HTTPHeaderCode::HTTP_HEADER_SET_COOKIE, fmt::format("ses={}; SameSite=Strict; HttpOnly", *response.setCookie)); - } - - if(response.contentType) - { - respBuilder.header(HTTPHeaderCode::HTTP_HEADER_CONTENT_TYPE, *response.contentType); - } - - respBuilder.sendWithEOM(); - - self->finished_ = true; }); - } - ); - return; - } - if(request_type == RequestType::CompleteMultipartUpload) { if(finished_) diff --git a/s3handler.h b/s3handler.h index 46e28c1..50ca048 100644 --- a/s3handler.h +++ b/s3handler.h @@ -221,7 +221,6 @@ private: int readMultipartExt(int64_t offset); void readBodyThread(folly::EventBase *evb); bool setKeyInfoFromPath(const std::string_view path); - bool handleApiCall(proxygen::HTTPMessage& headers); std::optional initPayloadHash(proxygen::HTTPMessage& message); enum class RequestType @@ -268,6 +267,4 @@ private: std::mutex bodyMutex; bool hasBodyThread = false; std::queue bodyQueue; - - std::unique_ptr apiHandler; }; diff --git a/www/vite.config.ts b/www/vite.config.ts index a74b187..a462770 100644 --- a/www/vite.config.ts +++ b/www/vite.config.ts @@ -3,11 +3,22 @@ import react from "@vitejs/plugin-react"; // https://vitejs.dev/config/ export default defineConfig({ - base: "/admin-b64be512-4b03-4028-a589-13931942e205/", + base: "/", server: { host: "127.0.0.1", proxy: { "^((?!(\/admin-b64be512-4b03-4028-a589-13931942e205)).)*$": "http://localhost:11000", }, }, + build: { + rollupOptions: { + output: { + assetFileNames: (assetInfo) => { + return "admin-b64be5124b034028a58913931942e205/[name]-[hash].[extname]"; + }, + chunkFileNames: 'admin-b64be5124b034028a58913931942e205/js/[name]-[hash].js', + entryFileNames: 'admin-b64be5124b034028a58913931942e205/js/[name]-[hash].js' + } + } + } }); diff --git a/wwwgen/www_files.cpp b/wwwgen/www_files.cpp new file mode 100644 index 0000000..51a740d --- /dev/null +++ b/wwwgen/www_files.cpp @@ -0,0 +1,18 @@ + +unsigned char index_html[] = { + 0x1f, 0x8b, 0x08, 0x00, 0xa9, 0x2e, 0xa9, 0x67, 0x02, 0xff, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; +unsigned int index_html_len = 20; + +unsigned char favicon_ico[] = { + 0x1f, 0x8b, 0x08, 0x00, 0xa9, 0x2e, 0xa9, 0x67, 0x02, 0xff, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; +unsigned int favicon_ico_len = 20; + +unsigned char index_4363833e_js[] = { + 0x1f, 0x8b, 0x08, 0x00, 0xa9, 0x2e, 0xa9, 0x67, 0x02, 0xff, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; +unsigned int index_4363833e_js_len = 20; diff --git a/wwwgen/www_files.h b/wwwgen/www_files.h new file mode 100644 index 0000000..328240e --- /dev/null +++ b/wwwgen/www_files.h @@ -0,0 +1,27 @@ +#pragma once +#include +#include + +extern unsigned char index_html[]; +extern unsigned int index_html_len; + +extern unsigned char favicon_ico[]; +extern unsigned int favicon_ico_len; + +extern unsigned char index_4363833e_js[]; +extern unsigned int index_4363833e_js_len; + +struct WwwFile +{ + const unsigned char* data; + unsigned int len; + std::string etag; + bool immutable; + std::string_view contentType; + unsigned int uncompLen; +}; +static std::map www_files = { + { "index.html", WwwFile{ index_html, index_html_len, "\"d41d8cd98f00b204e9800998ecf8427e\"", false, "text/html", 0 } }, + { "favicon.ico", WwwFile{ favicon_ico, favicon_ico_len, "\"d41d8cd98f00b204e9800998ecf8427e\"", false, "image/x-icon", 0 } }, + { "admin-b64be5124b034028a58913931942e205/js/index-4363833e.js", WwwFile{ index_4363833e_js, index_4363833e_js_len, "\"d41d8cd98f00b204e9800998ecf8427e\"", false, "application/javascript", 0 } }, +}; diff --git a/wwwgen/wwwgen.py b/wwwgen/wwwgen.py new file mode 100644 index 0000000..c0edf72 --- /dev/null +++ b/wwwgen/wwwgen.py @@ -0,0 +1,110 @@ +from dataclasses import dataclass +import sys +import os +from pathlib import Path +import gzip +import hashlib + +@dataclass +class Sourcefile: + source: str + header: str + fn: Path + var_name: str + etag: str + uncomp_len: int + +def xxd_i(filename: Path, empty: bool) -> Sourcefile: + with open(filename, 'rb') as f: + raw_data = b"" if empty else f.read() + + etag = hashlib.md5(raw_data).hexdigest() + data = gzip.compress(raw_data) + + var_name = filename.name.replace('.', '_').replace('-', '_') + source = f"unsigned char {var_name}[] = {{\n" + header = f"extern unsigned char {var_name}[];\nextern unsigned int {var_name}_len;\n" + + for i in range(0, len(data), 12): + line = ', '.join(f'0x{byte:02x}' for byte in data[i:i+12]) + source += f" {line},\n" + + source += f"}};\nunsigned int {var_name}_len = {len(data)};\n" + return Sourcefile(source, header, filename, var_name, etag, len(raw_data)) + +def get_content_type(fn: Path) -> str: + if fn.suffix == ".html": + return "text/html" + elif fn.suffix == ".css": + return "text/css" + elif fn.suffix == ".js": + return "application/javascript" + elif fn.suffix == ".png": + return "image/png" + elif fn.suffix == ".jpg": + return "image/jpeg" + elif fn.suffix == ".jpeg": + return "image/jpeg" + elif fn.suffix == ".gif": + return "image/gif" + elif fn.suffix == ".svg": + return "image/svg+xml" + elif fn.suffix == ".ico": + return "image/x-icon" + elif fn.suffix == ".json": + return "application/json" + elif fn.suffix == ".xml": + return "application/xml" + elif fn.suffix == ".pdf": + return "application/pdf" + elif fn.suffix == ".zip": + return "application/zip" + + return "application/octet-stream" + + +def merge_sources(sources: list[Sourcefile], base_dir: Path) -> Sourcefile: + source = '' + header = '' + srcs = "struct WwwFile\n{\n\tconst unsigned char* data;\n\tunsigned int len;\n\tstd::string etag;\n\tbool immutable;\n\tstd::string_view contentType;\n\tunsigned int uncompLen;\n};\n" + srcs += "static std::map www_files = {\n" + for s in sources: + source += '\n' + s.source + header += '\n' + s.header + + rel_fn = str(s.fn)[len(str(base_dir))+1:] + + immutable = True if rel_fn.startswith("assets-") else False + imm_str = "true" if immutable else "false" + + srcs += f' {{ "{rel_fn}", WwwFile{{ {s.var_name}, {s.var_name}_len, "\\"{s.etag}\\"", {imm_str}, "{get_content_type(s.fn)}", {s.uncomp_len} }} }},\n' + + srcs += "};\n" + + return Sourcefile(source, "#pragma once\n#include \n#include \n" + header+"\n"+srcs, Path(), "", "", 0) + + +def list_files(dir: Path) -> list[Path]: + return [f for f in dir.rglob('*') if f.is_file()] + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python wwwgen.py ") + sys.exit(1) + + dir = Path(sys.argv[1]) + empty = sys.argv[2] == "empty" + + files = list_files(dir) + srcs = list[Sourcefile]() + + for file in files: + srcs.append(xxd_i(file, empty)) + + res = merge_sources(srcs, dir) + + with open("www_files.h", 'w') as f: + f.write(res.header) + + with open("www_files.cpp", 'w') as f: + f.write(res.source)