mirror of
https://github.com/uroni/hs5.git
synced 2025-10-26 11:17:18 +00:00
Increase max key size to 1024
This commit is contained in:
Martin Raiber
parent
364b7e833c
commit
209fa5960b
74
external/lmdb/mdb.c
vendored
74
external/lmdb/mdb.c
vendored
@ -565,7 +565,7 @@ static txnid_t mdb_debug_start;
|
||||
* #MDB_DUPSORT data items must fit on a node in a regular page.
|
||||
*/
|
||||
#ifndef MDB_MAXKEYSIZE
|
||||
#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511)
|
||||
#define MDB_MAXKEYSIZE 0
|
||||
#endif
|
||||
|
||||
/** The maximum size of a key we can write to the environment. */
|
||||
@ -10444,51 +10444,51 @@ int mdb_cursor_next_leaf_page(MDB_cursor* cursor, size_t* pgno)
|
||||
*pgno = cursor->mc_pg[cursor->mc_top]->mp_pgno;
|
||||
|
||||
return MDB_SUCCESS;
|
||||
}
|
||||
|
||||
int mdb_cursor_get_pageno(MDB_cursor* cursor, size_t* pgno)
|
||||
{
|
||||
}
|
||||
|
||||
int mdb_cursor_get_pageno(MDB_cursor* cursor, size_t* pgno)
|
||||
{
|
||||
if (pgno == NULL)
|
||||
return MDB_INVALID;
|
||||
|
||||
return MDB_INVALID;
|
||||
|
||||
*pgno = cursor->mc_pg[cursor->mc_top]->mp_pgno;
|
||||
|
||||
return MDB_SUCCESS;
|
||||
}
|
||||
|
||||
int mdb_page_get_nkeys(MDB_cursor* mc, size_t pgno, unsigned int* nkeys)
|
||||
{
|
||||
if (nkeys == NULL)
|
||||
return MDB_INVALID;
|
||||
|
||||
MDB_page* pg;
|
||||
int rc = mdb_page_get(mc, pgno, &pg, NULL);
|
||||
if (rc != MDB_SUCCESS)
|
||||
return rc;
|
||||
|
||||
if (!IS_LEAF(pg))
|
||||
return MDB_CORRUPTED;
|
||||
|
||||
if (IS_LEAF2(pg))
|
||||
return MDB_CORRUPTED;
|
||||
|
||||
*nkeys = NUMKEYS(pg);
|
||||
|
||||
return MDB_SUCCESS;
|
||||
return MDB_SUCCESS;
|
||||
}
|
||||
|
||||
int mdb_page_get_nkeys(MDB_cursor* mc, size_t pgno, unsigned int* nkeys)
|
||||
{
|
||||
if (nkeys == NULL)
|
||||
return MDB_INVALID;
|
||||
|
||||
MDB_page* pg;
|
||||
int rc = mdb_page_get(mc, pgno, &pg, NULL);
|
||||
if (rc != MDB_SUCCESS)
|
||||
return rc;
|
||||
|
||||
if (!IS_LEAF(pg))
|
||||
return MDB_CORRUPTED;
|
||||
|
||||
if (IS_LEAF2(pg))
|
||||
return MDB_CORRUPTED;
|
||||
|
||||
*nkeys = NUMKEYS(pg);
|
||||
|
||||
return MDB_SUCCESS;
|
||||
}
|
||||
|
||||
int mdb_page_get_val(MDB_cursor* mc, size_t pgno, unsigned int idx,
|
||||
MDB_val* key, MDB_val* data)
|
||||
{
|
||||
MDB_page* pg;
|
||||
int rc = mdb_page_get(mc, pgno, &pg, NULL);
|
||||
if (rc != MDB_SUCCESS)
|
||||
MDB_page* pg;
|
||||
int rc = mdb_page_get(mc, pgno, &pg, NULL);
|
||||
if (rc != MDB_SUCCESS)
|
||||
return rc;
|
||||
|
||||
if (!IS_LEAF(pg))
|
||||
if (!IS_LEAF(pg))
|
||||
return MDB_CORRUPTED;
|
||||
|
||||
if (IS_LEAF2(pg))
|
||||
if (IS_LEAF2(pg))
|
||||
return MDB_CORRUPTED;
|
||||
|
||||
unsigned int nkeys = NUMKEYS(pg);
|
||||
@ -10509,9 +10509,9 @@ int mdb_page_get_val(MDB_cursor* mc, size_t pgno, unsigned int idx,
|
||||
|
||||
int mdb_page_is_dirty(MDB_cursor* mc, size_t pgno, int* dirty)
|
||||
{
|
||||
MDB_page* pg;
|
||||
int rc = mdb_page_get(mc, pgno, &pg, NULL);
|
||||
if (rc != MDB_SUCCESS)
|
||||
MDB_page* pg;
|
||||
int rc = mdb_page_get(mc, pgno, &pg, NULL);
|
||||
if (rc != MDB_SUCCESS)
|
||||
return rc;
|
||||
|
||||
*dirty = ((char*)pg < mc->mc_txn->mt_env->me_map ||
|
||||
|
||||
@ -39,6 +39,9 @@
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
// Max number of bytes in file paths. S3 max is 1024, but leave 1 + 8 + 8 bytes for versioning
|
||||
const size_t c_max_path = 1024 + 1 + 8 + 8;
|
||||
|
||||
DEFINE_bool(symlink_lockfile_to_tmpdir, false, "Symlink LMDB lockfile to /tmp");
|
||||
|
||||
#ifndef _WIN32
|
||||
@ -1349,7 +1352,7 @@ int SingleFileStorage::write(const std::string & fn, const char* data,
|
||||
return ENODEV;
|
||||
}
|
||||
|
||||
if (fn.size() > 255)
|
||||
if (fn.size() > c_max_path)
|
||||
return EINVAL;
|
||||
|
||||
return write_int(fn, data, data_size, data_alloc_size, last_modified, md5sum, true, no_del_old);
|
||||
@ -1665,7 +1668,7 @@ SingleFileStorage::WritePrepareResult SingleFileStorage::write_prepare(const std
|
||||
return WritePrepareResult{ENODEV};
|
||||
}
|
||||
|
||||
if (fn.size() > 255)
|
||||
if (fn.size() > c_max_path)
|
||||
return WritePrepareResult{EINVAL};
|
||||
|
||||
if(data_size==0)
|
||||
|
||||
@ -1649,6 +1649,20 @@ void S3Handler::putObject(proxygen::HTTPMessage& headers)
|
||||
if(self->withBucketVersioning)
|
||||
self->keyInfo.version = self->sfs.get_next_version();
|
||||
|
||||
if(self->keyInfo.key.size()>1024)
|
||||
{
|
||||
evb->runInEventBaseThread([self = self]()
|
||||
{
|
||||
ResponseBuilder(self->downstream_)
|
||||
.status(500, "Internal error")
|
||||
.body(fmt::format("Key too long"))
|
||||
.sendWithEOM();
|
||||
std::lock_guard lock(self->extents_mutex);
|
||||
self->finished_ = true;
|
||||
self->extents_cond.notify_all(); });
|
||||
return;
|
||||
}
|
||||
|
||||
const auto fpath = make_key(self->keyInfo);
|
||||
|
||||
auto res = self->sfs.write_prepare(fpath, self->put_remaining);
|
||||
@ -2920,7 +2934,7 @@ void S3Handler::onBodyCPU(folly::EventBase *evb, int64_t offset, std::unique_ptr
|
||||
{
|
||||
evb->runInEventBaseThread([self = this]()
|
||||
{
|
||||
if(!self)
|
||||
if(!self || self->finished_)
|
||||
return;
|
||||
ResponseBuilder(self->downstream_)
|
||||
.status(500, "Internal error")
|
||||
@ -2940,7 +2954,7 @@ void S3Handler::onBodyCPU(folly::EventBase *evb, int64_t offset, std::unique_ptr
|
||||
{
|
||||
evb->runInEventBaseThread([self = this]()
|
||||
{
|
||||
if(!self)
|
||||
if(!self || self->finished_)
|
||||
return;
|
||||
ResponseBuilder(self->downstream_)
|
||||
.status(400, "Bad request")
|
||||
@ -2965,7 +2979,7 @@ void S3Handler::onBodyCPU(folly::EventBase *evb, int64_t offset, std::unique_ptr
|
||||
{
|
||||
evb->runInEventBaseThread([self = this]()
|
||||
{
|
||||
if(!self)
|
||||
if(!self || self->finished_)
|
||||
return;
|
||||
ResponseBuilder(self->downstream_)
|
||||
.status(500, "Internal error")
|
||||
@ -2983,7 +2997,7 @@ void S3Handler::onBodyCPU(folly::EventBase *evb, int64_t offset, std::unique_ptr
|
||||
{
|
||||
evb->runInEventBaseThread([self = this]()
|
||||
{
|
||||
if(!self)
|
||||
if(!self || self->finished_)
|
||||
return;
|
||||
ResponseBuilder(self->downstream_)
|
||||
.status(500, "Internal error")
|
||||
@ -3072,14 +3086,12 @@ void S3Handler::onBodyCPU(folly::EventBase *evb, int64_t offset, std::unique_ptr
|
||||
{
|
||||
evb->runInEventBaseThread([self = this->self]()
|
||||
{
|
||||
if(!self)
|
||||
return;
|
||||
if(!self->finished_)
|
||||
ResponseBuilder(self->downstream_)
|
||||
.status(500, "Internal error")
|
||||
.body("Write ext error")
|
||||
.sendWithEOM();
|
||||
|
||||
if(!self || self->finished_)
|
||||
return;
|
||||
ResponseBuilder(self->downstream_)
|
||||
.status(500, "Internal error")
|
||||
.body("Write ext error")
|
||||
.sendWithEOM();
|
||||
self->finished_ = true; });
|
||||
return;
|
||||
}
|
||||
|
||||
@ -435,4 +435,35 @@ def test_create_bucket(hs5: Hs5Runner, tmp_path: Path):
|
||||
|
||||
s3_client.delete_object(Bucket=bucketname, Key="upload.txt")
|
||||
|
||||
s3_client.delete_bucket(Bucket=bucketname)
|
||||
s3_client.delete_bucket(Bucket=bucketname)
|
||||
|
||||
|
||||
def test_upload_long_key(hs5: Hs5Runner, tmp_path: Path):
|
||||
s3_client = hs5.get_s3_client()
|
||||
|
||||
key = "a" * 1024
|
||||
with open(tmp_path / "upload.txt", "w") as upload_file:
|
||||
upload_file.write("abc")
|
||||
|
||||
s3_client.upload_file(upload_file.name, hs5.testbucketname(), key)
|
||||
|
||||
with pytest.raises(ClientError):
|
||||
with io.FileIO(tmp_path / "upload.txt", "rb") as upload_file:
|
||||
s3_client.put_object(Bucket=hs5.testbucketname(), Key= "a" * 1025, Body=upload_file)
|
||||
|
||||
dl_path = tmp_path / "download.txt"
|
||||
s3_client.download_file(hs5.testbucketname(), key, str(dl_path))
|
||||
assert os.stat(dl_path).st_size == 3
|
||||
|
||||
list = s3_client.list_objects(Bucket=hs5.testbucketname())
|
||||
assert not list["IsTruncated"]
|
||||
assert "Contents" in list
|
||||
objs = list["Contents"]
|
||||
assert len(objs) == 1
|
||||
assert "Key" in objs[0] and objs[0]["Key"] == key
|
||||
|
||||
s3_client.delete_object(Bucket=hs5.testbucketname(), Key=key)
|
||||
|
||||
list = s3_client.list_objects(Bucket=hs5.testbucketname())
|
||||
assert not list["IsTruncated"]
|
||||
assert "Contents" not in list
|
||||
|
||||
Loading…
Reference in New Issue
Block a user