Archives/falk-werner_webfuse
1
0
Fork 0
mirror of https://github.com/falk-werner/webfuse synced 2024-10-27 20:34:10 +00:00
Code Issues Releases Wiki Activity

Merge pull request #84 from falk-werner/large_file_tests

Browse Source 
Large file tests
This commit is contained in:
Falk Werner 2020-07-06 19:41:52 +02:00 committed by GitHub
commit 15d8425d94
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 190 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
changelog.md
View File

@ -2,6 +2,10 @@
## 0.5.0 _(unknown)_
### Fixes
* Fix deadlock reading files larger that a few KBytes (Isseue #82)
## 0.4.0 _(Sun Jul 05 2020)_
### Breaking Changes

11
lib/webfuse/impl/operation/read.c
View File

@ -10,7 +10,8 @@
#include "webfuse/impl/util/base64.h"
#include "webfuse/impl/util/json_util.h"
#define WF_MAX_READ_LENGTH 4096
// do not read chunks larger than 1 MByte
#define WF_MAX_READ_LENGTH (1024 * 1024)
char * wf_impl_fill_buffer(
char const * data,
@ -114,12 +115,16 @@ void wf_impl_operation_read(
struct wf_impl_operation_context * user_data = fuse_req_userdata(request);
struct wf_jsonrpc_proxy * rpc = wf_impl_operation_context_get_proxy(user_data);
if (NULL != rpc)
if ((NULL != rpc) && (size <= WF_MAX_READ_LENGTH))
{
int const length = (size <= WF_MAX_READ_LENGTH) ? (int) size : WF_MAX_READ_LENGTH;
int const length = (int) size;
int handle = (file_info->fh & INT_MAX);
wf_impl_jsonrpc_proxy_invoke(rpc, &wf_impl_operation_read_finished, request, "read", "siiii", user_data->name, (int) inode, handle, (int) offset, length);
}
else if (size > WF_MAX_READ_LENGTH)
{
fuse_reply_err(request, ENOMEM);
}
else
{
fuse_reply_err(request, ENOENT);

26
test/webfuse/fs_check/main.c
View File

@ -70,6 +70,7 @@ static bool print_usage(
"\thas_size - checks, if <file> has the size given in <arg>\n"
"\thas_subdir - checks, if <file> contains the sub directory given in <arg>\n"
"\thas_contents - checks, if <file> has the contents given in <arg>\n"
"\tread_all - checks, if alls contents of <file> can be read\n"
);
return command->success;
@ -159,6 +160,30 @@ static bool has_contents(
return result;
}
static bool read_all(
struct command * command)
{
bool result = false;
struct stat info;
int rc = stat(command->file, &info);
if (0 != rc) { return false; }
size_t length = info.st_size;
char * buffer = malloc(length);
FILE * file = fopen(command->file, "rb");
{
ssize_t count = fread(buffer, 1, length, file);
fclose(file);
result = (count == (ssize_t) length);
}
free(buffer);
return result;
}
static bool get_command(
struct command * command,
char const * name)
@ -175,6 +200,7 @@ static bool get_command(
{"has_size" , &has_size , true},
{"has_subdir" , &has_subdir , true},
{"has_contents", &has_contents, true},
{"read_all" , &read_all , false},
{NULL, NULL, false}
};

1
test/webfuse/mocks/getattr_matcher.hpp
View File

@ -26,6 +26,7 @@ MATCHER_P(GetAttr, inode, "")
{
*result_listener << "inode mismatch: expected" << inode
<< " but was " << json_integer_value(inode_);
return false;
}
return true;

1
test/webfuse/mocks/open_matcher.hpp
View File

@ -26,6 +26,7 @@ MATCHER_P(Open, inode, "")
{
*result_listener << "inode mismatch: expected" << inode
<< " but was " << json_integer_value(inode_);
return false;
}
return true;

1
test/webfuse/mocks/readdir_matcher.hpp
View File

@ -26,6 +26,7 @@ MATCHER_P(ReadDir, inode, "")
{
*result_listener << "inode mismatch: expected" << inode
<< " but was " << json_integer_value(inode_);
return false;
}
return true;

3
test/webfuse/operation/test_read.cc
View File

@ -39,7 +39,7 @@ TEST(wf_impl_operation_read, invoke_proxy)
TEST(wf_impl_operation_read, invoke_proxy_limit_size)
{
MockJsonRpcProxy proxy;
EXPECT_CALL(proxy, wf_impl_jsonrpc_proxy_vinvoke(_,_,_,StrEq("read"),StrEq("siiii"))).Times(1);
EXPECT_CALL(proxy, wf_impl_jsonrpc_proxy_vinvoke(_,_,_,StrEq("read"),StrEq("siiii"))).Times(0);
MockOperationContext context;
EXPECT_CALL(context, wf_impl_operation_context_get_proxy(_)).Times(1)
@ -49,6 +49,7 @@ TEST(wf_impl_operation_read, invoke_proxy_limit_size)
op_context.name = nullptr;
FuseMock fuse;
EXPECT_CALL(fuse, fuse_req_userdata(_)).Times(1).WillOnce(Return(&op_context));
EXPECT_CALL(fuse, fuse_reply_err(_,_)).Times(1);
fuse_req_t request = nullptr;
fuse_ino_t inode = 1;

78
test/webfuse/test_client.cc
View File

@ -549,7 +549,7 @@ TEST(AdapterClient, LookupFile)
ASSERT_EQ(std::future_status::ready, disconnected.get_future().wait_for(TIMEOUT));
}
TEST(AdapterClient, ReadLargeFile)
TEST(AdapterClient, ReadFile)
{
MockInvokationHander handler;
WsServer server(handler, WF_PROTOCOL_NAME_PROVIDER_SERVER);
@ -561,6 +561,8 @@ TEST(AdapterClient, ReadLargeFile)
.WillOnce(Return("{\"inode\": 2, \"mode\": 420, \"type\": \"file\", \"size\": 4096}"));
EXPECT_CALL(handler, Invoke(StrEq("getattr"), GetAttr(1))).Times(AnyNumber())
.WillRepeatedly(Return("{\"mode\": 420, \"type\": \"dir\"}"));
EXPECT_CALL(handler, Invoke(StrEq("getattr"), GetAttr(2))).Times(AnyNumber())
.WillRepeatedly(Return("{\"mode\": 420, \"type\": \"file\", \"size\": 4096}"));
EXPECT_CALL(handler, Invoke(StrEq("open"), Open(2))).Times(1)
.WillOnce(Return("{\"handle\": 42}"));
EXPECT_CALL(handler, Invoke(StrEq("read"), _)).Times(AnyNumber())
@ -621,3 +623,77 @@ TEST(AdapterClient, ReadLargeFile)
client.Disconnect();
ASSERT_EQ(std::future_status::ready, disconnected.get_future().wait_for(TIMEOUT));
}
TEST(AdapterClient, ReadLargeFile)
{
MockInvokationHander handler;
WsServer server(handler, WF_PROTOCOL_NAME_PROVIDER_SERVER);
EXPECT_CALL(handler, Invoke(StrEq("add_filesystem"),_)).Times(1)
.WillOnce(Return("{\"id\": \"test\"}"));
EXPECT_CALL(handler, Invoke(StrEq("lookup"), _)).Times(AnyNumber())
.WillRepeatedly(Throw(std::runtime_error("unknown")));
EXPECT_CALL(handler, Invoke(StrEq("lookup"), Lookup(1, "a.file"))).Times(1)
.WillOnce(Return("{\"inode\": 2, \"mode\": 420, \"type\": \"file\", \"size\": 102400}"));
EXPECT_CALL(handler, Invoke(StrEq("getattr"), GetAttr(1))).Times(AnyNumber())
.WillRepeatedly(Return("{\"mode\": 420, \"type\": \"dir\"}"));
EXPECT_CALL(handler, Invoke(StrEq("getattr"), GetAttr(2))).Times(AnyNumber())
.WillRepeatedly(Return("{\"mode\": 420, \"type\": \"file\", \"size\": 102400}"));
EXPECT_CALL(handler, Invoke(StrEq("open"), Open(2))).Times(1)
.WillOnce(Return("{\"handle\": 42}"));
EXPECT_CALL(handler, Invoke(StrEq("read"), _)).Times(AnyNumber())
.WillRepeatedly(Invoke([](char const *, json_t * params) {
int offset = json_integer_value(json_array_get(params, 3));
int length = json_integer_value(json_array_get(params, 4));
int remaining = (offset < 102400) ? 102400 - offset : 0;
int count = (length < remaining) ? length : remaining;
std::string data = std::string(count, '*');
json_t * result = json_object();
json_object_set_new(result, "data", json_string(data.c_str()));
json_object_set_new(result, "format", json_string("identity"));
json_object_set_new(result, "count", json_integer(count));
char * result_text = json_dumps(result, 0);
std::string result_str = result_text;
free(result_text);
json_decref(result);
return result_str;
}));
EXPECT_CALL(handler, Invoke(StrEq("close"), _)).Times(AtMost(1));
MockAdapterClientCallback callback;
EXPECT_CALL(callback, Invoke(_, _, _)).Times(AnyNumber());
std::promise<void> connected;
EXPECT_CALL(callback, Invoke(_, WF_CLIENT_CONNECTED, nullptr)).Times(1)
.WillOnce(Invoke([&] (wf_client *, int, void *) mutable { connected.set_value(); }));
std::promise<void> disconnected;
EXPECT_CALL(callback, Invoke(_, WF_CLIENT_DISCONNECTED, nullptr)).Times(1)
.WillOnce(Invoke([&] (wf_client *, int, void *) mutable { disconnected.set_value(); }));
std::promise<void> called;
EXPECT_CALL(callback, Invoke(_, WF_CLIENT_FILESYSTEM_ADDED, nullptr)).Times(1)
.WillOnce(Invoke([&called] (wf_client *, int, void *) mutable {
called.set_value();
}));
AdapterClient client(callback.GetCallbackFn(), callback.GetUserData(), server.GetUrl());
client.Connect();
ASSERT_EQ(std::future_status::ready, connected.get_future().wait_for(TIMEOUT));
client.AddFileSystem();
ASSERT_EQ(std::future_status::ready, called.get_future().wait_for(TIMEOUT));
std::string base_dir = client.GetDir();
ASSERT_TRUE(File(base_dir).isDirectory());
File file(base_dir + "/a.file");
ASSERT_TRUE(file.readAll());
client.Disconnect();
ASSERT_EQ(std::future_status::ready, disconnected.get_future().wait_for(TIMEOUT));
}

62
test/webfuse/test_server.cc
View File

@ -316,7 +316,7 @@ TEST(server, read)
ASSERT_TRUE(disconnected);
}
TEST(server, read_large_file)
TEST(server, read_large_file_contents)
{
Server server;
MockInvokationHander handler;
@ -325,6 +325,8 @@ TEST(server, read_large_file)
.WillOnce(Return("{\"inode\": 2, \"mode\": 420, \"type\": \"file\", \"size\": 4096}"));
EXPECT_CALL(handler, Invoke(StrEq("getattr"), GetAttr(1))).Times(AnyNumber())
.WillRepeatedly(Return("{\"mode\": 420, \"type\": \"dir\"}"));
EXPECT_CALL(handler, Invoke(StrEq("getattr"), GetAttr(2))).Times(AnyNumber())
.WillRepeatedly(Return("{\"mode\": 420, \"type\": \"file\", \"size\": 4096}"));
EXPECT_CALL(handler, Invoke(StrEq("open"), Open(2))).Times(1)
.WillOnce(Return("{\"handle\": 42}"));
EXPECT_CALL(handler, Invoke(StrEq("read"), _)).Times(AnyNumber())
@ -374,6 +376,64 @@ TEST(server, read_large_file)
ASSERT_TRUE(disconnected);
}
TEST(server, read_large_file)
{
Server server;
MockInvokationHander handler;
EXPECT_CALL(handler, Invoke(StrEq("lookup"), _)).Times(AnyNumber());
EXPECT_CALL(handler, Invoke(StrEq("lookup"), Lookup(1, "a.file"))).Times(1)
.WillOnce(Return("{\"inode\": 2, \"mode\": 420, \"type\": \"file\", \"size\": 102400}"));
EXPECT_CALL(handler, Invoke(StrEq("getattr"), GetAttr(1))).Times(AnyNumber())
.WillRepeatedly(Return("{\"mode\": 420, \"type\": \"dir\"}"));
EXPECT_CALL(handler, Invoke(StrEq("getattr"), GetAttr(2))).Times(AnyNumber())
.WillRepeatedly(Return("{\"mode\": 420, \"type\": \"file\", \"size\": 102400}"));
EXPECT_CALL(handler, Invoke(StrEq("open"), Open(2))).Times(1)
.WillOnce(Return("{\"handle\": 42}"));
EXPECT_CALL(handler, Invoke(StrEq("read"), _)).Times(AnyNumber())
.WillRepeatedly(Invoke([](char const *, json_t * params) {
int offset = json_integer_value(json_array_get(params, 3));
int length = json_integer_value(json_array_get(params, 4));
int remaining = (offset < 102400) ? 102400 - offset : 0;
int count = (length < remaining) ? length : remaining;
std::string data = std::string(count, '*');
json_t * result = json_object();
json_object_set_new(result, "data", json_string(data.c_str()));
json_object_set_new(result, "format", json_string("identity"));
json_object_set_new(result, "count", json_integer(count));
char * result_text = json_dumps(result, 0);
std::string result_str = result_text;
free(result_text);
json_decref(result);
return result_str;
}));
EXPECT_CALL(handler, Invoke(StrEq("close"), _)).Times(AtMost(1));
WsClient client(handler, WF_PROTOCOL_NAME_PROVIDER_CLIENT);
auto connected = client.Connect(server.GetPort(), WF_PROTOCOL_NAME_ADAPTER_SERVER);
ASSERT_TRUE(connected);
std::string response_text = client.Invoke("{\"method\": \"add_filesystem\", \"params\": [\"test\"], \"id\": 42}");
json_t * response = json_loads(response_text.c_str(), 0, nullptr);
ASSERT_TRUE(json_is_object(response));
json_t * result = json_object_get(response, "result");
ASSERT_TRUE(json_is_object(result));
json_t * id = json_object_get(response, "id");
ASSERT_EQ(42, json_integer_value(id));
json_decref(response);
std::string base_dir = server.GetBaseDir();
ASSERT_TRUE(File(base_dir).isDirectory());
File file(base_dir + "/test/a.file");
ASSERT_TRUE(file.readAll());
auto disconnected = client.Disconnect();
ASSERT_TRUE(disconnected);
}
TEST(server, readdir)
{

8
test/webfuse/test_util/file.cc
View File

@ -82,4 +82,12 @@ bool File::hasContents(std::string const & contents)
return invoke(command.str());
}
bool File::readAll()
{
std::stringstream command;
command << "./fs_check -c read_all -f " << path_;
return invoke(command.str());
}
}

1
test/webfuse/test_util/file.hpp
View File

@ -17,6 +17,7 @@ public:
bool hasSize(size_t size);
bool hasSubdirectory(std::string const & subdir);
bool hasContents(std::string const & contents);
bool readAll();
private:
std::string path_;
};