agorise/c-ipfs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

misc tweaks and fixes

Browse source 
Now paying attention to the IPFS_PATH environment variable to determine
where the repository is. Fixed some broken tests. Fixed a bug whereby a
subdirectory within a subdirectory was not displaying correctly when
imported.
This commit is contained in:
jmjatlanta 2017-01-01 23:48:09 -05:00
parent e1582544b1
commit 61d0adc445
12 changed files with 168 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
importer/exporter.c
View file

@ -38,31 +38,43 @@ int ipfs_exporter_to_file(const unsigned char* hash, const char* file_name, cons
ipfs_node_free(read_node); ipfs_node_free(read_node);
return 0; return 0;
} }
size_t bytes_written = fwrite(read_node->data, 1, read_node->data_size, file);
if (bytes_written != read_node->data_size) {
fclose(file);
ipfs_node_free(read_node);
return 0;
}
struct NodeLink* link = read_node->head_link;
struct Node* link_node = NULL;
while (link != NULL) {
if ( ipfs_merkledag_get(link->hash, link->hash_size, &link_node, fs_repo) == 0) {
fclose(file);
ipfs_node_free(read_node);
return 0;
}
bytes_written = fwrite(link_node->data, 1, link_node->data_size, file);
if (bytes_written != link_node->data_size) {
ipfs_node_free(link_node);
fclose(file);
ipfs_node_free(read_node);
return 0;
}
ipfs_node_free(link_node);
link = link->next;
}
if (read_node->head_link == NULL) {
// convert the node's data into a UnixFS data block
struct UnixFS* unix_fs;
ipfs_unixfs_protobuf_decode(read_node->data, read_node->data_size, &unix_fs);
size_t bytes_written = fwrite(unix_fs->bytes, 1, unix_fs->bytes_size, file);
if (bytes_written != unix_fs->bytes_size) {
fclose(file);
ipfs_node_free(read_node);
ipfs_unixfs_free(unix_fs);
return 0;
}
ipfs_unixfs_free(unix_fs);
} else {
struct NodeLink* link = read_node->head_link;
struct Node* link_node = NULL;
while (link != NULL) {
if ( ipfs_merkledag_get(link->hash, link->hash_size, &link_node, fs_repo) == 0) {
fclose(file);
ipfs_node_free(read_node);
return 0;
}
struct UnixFS* unix_fs;
ipfs_unixfs_protobuf_decode(link_node->data, link_node->data_size, &unix_fs);
size_t bytes_written = fwrite(unix_fs->bytes, 1, unix_fs->bytes_size, file);
if (bytes_written != unix_fs->bytes_size) {
ipfs_node_free(link_node);
fclose(file);
ipfs_node_free(read_node);
ipfs_unixfs_free(unix_fs);
return 0;
}
ipfs_node_free(link_node);
ipfs_unixfs_free(unix_fs);
link = link->next;
}
}
fclose(file); fclose(file);
if (read_node != NULL) if (read_node != NULL)

100
importer/importer.c
View file

@ -178,11 +178,15 @@ int ipfs_import_print_node_results(const struct Node* node, const char* file_nam
/** /**
* Creates a node based on an incoming file or directory * Creates a node based on an incoming file or directory
* NOTE: this can be called recursively for directories * NOTE: this can be called recursively for directories
* @param root_dir the directory for where to look for the file
* @param file_name the file (or directory) to import * @param file_name the file (or directory) to import
* @param parent_node the root node (has links to others in case this is a large file and is split) * @param parent_node the root node (has links to others in case this is a large file and is split)
* @param fs_repo the ipfs repository
* @param bytes_written number of bytes written to disk
* @param recursive true if we should navigate directories
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** parent_node, struct FSRepo* fs_repo, size_t* bytes_written) { int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** parent_node, struct FSRepo* fs_repo, size_t* bytes_written, int recursive) {
/** /**
* NOTE: When this function completes, parent_node will be either: * NOTE: When this function completes, parent_node will be either:
* 1) the complete file, in the case of a small file (<256k-ish) * 1) the complete file, in the case of a small file (<256k-ish)
@ -194,44 +198,76 @@ int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** p
size_t total_size = 0; size_t total_size = 0;
if (os_utils_is_directory(fileName)) { if (os_utils_is_directory(fileName)) {
// calculate the new root_dir
char* new_root_dir = (char*)root_dir;
char* path = NULL;
char* file = NULL;
os_utils_split_filename(fileName, &path, &file);
if (root_dir == NULL) {
new_root_dir = file;
} else {
free(path);
path = malloc(strlen(root_dir) + strlen(file) + 2);
os_utils_filepath_join(root_dir, file, path, strlen(root_dir) + strlen(file) + 2);
new_root_dir = path;
}
// initialize parent_node as a directory // initialize parent_node as a directory
if (ipfs_node_create_directory(parent_node) == 0) if (ipfs_node_create_directory(parent_node) == 0) {
if (path != NULL)
free(path);
if (file != NULL)
free(file);
return 0; return 0;
}
// get list of files // get list of files
struct FileList* first = os_utils_list_directory(fileName); struct FileList* first = os_utils_list_directory(fileName);
struct FileList* next = first; struct FileList* next = first;
while (next != NULL) { if (recursive) {
// process each file. NOTE: could be an embedded directory while (next != NULL) {
*bytes_written = 0; // process each file. NOTE: could be an embedded directory
struct Node* file_node; *bytes_written = 0;
size_t filename_len = strlen(fileName) + strlen(next->file_name) + 2; struct Node* file_node;
char full_file_name[filename_len]; // put the filename together from fileName, which is the directory, and next->file_name
os_utils_filepath_join(fileName, next->file_name, full_file_name, filename_len); // which is a file (or a directory) within the directory we just found.
if (ipfs_import_file(root_dir, full_file_name, &file_node, fs_repo, bytes_written) == 0) { size_t filename_len = strlen(fileName) + strlen(next->file_name) + 2;
ipfs_node_free(*parent_node); char full_file_name[filename_len];
os_utils_free_file_list(first); os_utils_filepath_join(fileName, next->file_name, full_file_name, filename_len);
return 0; // adjust root directory
}
// TODO: probably need to display what was imported if (ipfs_import_file(new_root_dir, full_file_name, &file_node, fs_repo, bytes_written, recursive) == 0) {
int len = strlen(next->file_name) + strlen(root_dir) + 2; ipfs_node_free(*parent_node);
char full_path[len]; os_utils_free_file_list(first);
os_utils_filepath_join(root_dir, next->file_name, full_path, len); if (file != NULL)
ipfs_import_print_node_results(file_node, full_path); free(file);
// TODO: Determine what needs to be done if this file_node is a file, a split file, or a directory if (path != NULL)
// Create link from file_node free (path);
struct NodeLink* file_node_link; return 0;
ipfs_node_link_create(next->file_name, file_node->hash, file_node->hash_size, &file_node_link); }
file_node_link->t_size = *bytes_written; // TODO: probably need to display what was imported
// add file_node as link to parent_node int len = strlen(next->file_name) + strlen(new_root_dir) + 2;
ipfs_node_add_link(*parent_node, file_node_link); char full_path[len];
// clean up file_node os_utils_filepath_join(new_root_dir, next->file_name, full_path, len);
ipfs_node_free(file_node); ipfs_import_print_node_results(file_node, full_path);
// move to next file in list // TODO: Determine what needs to be done if this file_node is a file, a split file, or a directory
next = next->next; // Create link from file_node
} // while going through files struct NodeLink* file_node_link;
ipfs_node_link_create(next->file_name, file_node->hash, file_node->hash_size, &file_node_link);
file_node_link->t_size = *bytes_written;
// add file_node as link to parent_node
ipfs_node_add_link(*parent_node, file_node_link);
// clean up file_node
ipfs_node_free(file_node);
// move to next file in list
next = next->next;
} // while going through files
}
// save the parent_node (the directory) // save the parent_node (the directory)
size_t bytes_written; size_t bytes_written;
ipfs_merkledag_add(*parent_node, fs_repo, &bytes_written); ipfs_merkledag_add(*parent_node, fs_repo, &bytes_written);
if (file != NULL)
free(file);
if (path != NULL)
free (path);
} else { } else {
// process this file // process this file
FILE* file = fopen(fileName, "rb"); FILE* file = fopen(fileName, "rb");
@ -304,7 +340,7 @@ int ipfs_import_files(int argc, char** argv) {
char* filename; char* filename;
os_utils_split_filename(current->file_name, &path, &filename); os_utils_split_filename(current->file_name, &path, &filename);
size_t bytes_written = 0; size_t bytes_written = 0;
retVal = ipfs_import_file(filename, current->file_name, &directory_entry, fs_repo, &bytes_written); retVal = ipfs_import_file(NULL, current->file_name, &directory_entry, fs_repo, &bytes_written, recursive);
ipfs_import_print_node_results(directory_entry, filename); ipfs_import_print_node_results(directory_entry, filename);
// cleanup // cleanup

8
include/ipfs/importer/importer.h
View file

@ -6,11 +6,15 @@
/** /**
* Creates a node based on an incoming file * Creates a node based on an incoming file
* @param file_name the file to import * @param root the root directory
* @param file_name the file to import (could contain a directory)
* @param node the root node (could have links to others) * @param node the root node (could have links to others)
* @param fs_repo the repo to use
* @param bytes_written the number of bytes written to disk
* @param recursive true(1) if you want to include files and directories
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_import_file(const char* root, const char* fileName, struct Node** node, struct FSRepo* fs_repo, size_t* bytes_written); int ipfs_import_file(const char* root, const char* fileName, struct Node** node, struct FSRepo* fs_repo, size_t* bytes_written, int recursive);
/** /**
* called from the command line * called from the command line

2
include/ipfs/os/utils.h
View file

@ -46,7 +46,7 @@ int os_utils_split_filename(const char* in, char** path, char** filename);
* @returns the results * @returns the results
*/ */
char* os_utils_getenv(const char* variable); char* os_utils_getenv(const char* variable);
int os_utils_setenv(const char* variable, const char* value, int overwrite);
/** /**
* get the user's home directory * get the user's home directory
* @returns the user's home directory * @returns the user's home directory

3
merkledag/merkledag.c
View file

@ -76,6 +76,9 @@ int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node*
return 0; return 0;
} }
// set the hash
ipfs_node_set_hash(*node, hash, hash_size);
return 1; return 1;
} }

11
os/utils.c
View file

@ -17,6 +17,17 @@ char* os_utils_getenv(const char* variable) {
return getenv(variable); return getenv(variable);
} }
/**
* set an environment variable in the os
* @param variable the variable to set
* @param value the value to assign to the variable
* @param overwrite passing a non-zero will allow an existing value to be overwritten
* @returns true(1) on success
*/
int os_utils_setenv(const char* variable, const char* value, int overwrite) {
return setenv(variable, value, overwrite);
}
/** /**
* returns the user's home directory * returns the user's home directory
* @returns the home directory * @returns the home directory

18
repo/fsrepo/fs_repo.c
View file

@ -116,15 +116,27 @@ int ipfs_repo_fsrepo_new(const char* repo_path, struct RepoConfig* config, struc
if (repo_path == NULL) { if (repo_path == NULL) {
// get the user's home directory // get the user's home directory
char* home_dir = os_utils_get_homedir(); char* ipfs_path = os_utils_getenv("IPFS_PATH");
if (ipfs_path == NULL)
ipfs_path = os_utils_get_homedir();
char* default_subdir = "/.ipfs"; char* default_subdir = "/.ipfs";
unsigned long newPathLen = strlen(home_dir) + strlen(default_subdir) + 2; // 1 for slash and 1 for end unsigned long newPathLen = 0;
if (strstr(ipfs_path, default_subdir) != NULL) {
newPathLen = strlen(ipfs_path) + 1;
} else {
// add /.ipfs to the string
newPathLen = strlen(ipfs_path) + strlen(default_subdir) + 2; // 1 for slash and 1 for end
}
(*repo)->path = malloc(sizeof(char) * newPathLen); (*repo)->path = malloc(sizeof(char) * newPathLen);
if ((*repo)->path == NULL) { if ((*repo)->path == NULL) {
free( (*repo)); free( (*repo));
return 0; return 0;
} }
os_utils_filepath_join(os_utils_get_homedir(), default_subdir, (*repo)->path, newPathLen); if (strstr(ipfs_path, default_subdir) != NULL) {
strcpy((*repo)->path, ipfs_path);
} else {
os_utils_filepath_join(os_utils_get_homedir(), default_subdir, (*repo)->path, newPathLen);
}
} else { } else {
int len = strlen(repo_path) + 1; int len = strlen(repo_path) + 1;
(*repo)->path = (char*)malloc(len); (*repo)->path = (char*)malloc(len);

6
repo/init.c
View file

@ -52,10 +52,16 @@ int make_ipfs_repository(const char* path) {
* Initialize a repository * Initialize a repository
*/ */
int ipfs_repo_init(int argc, char** argv) { int ipfs_repo_init(int argc, char** argv) {
// the default is the user's home directory
char* home_directory = os_utils_get_homedir(); char* home_directory = os_utils_get_homedir();
//allow user to pass directory on command line //allow user to pass directory on command line
if (argc > 2) { if (argc > 2) {
home_directory = argv[2]; home_directory = argv[2];
} else {
// check the IPFS_PATH
char* temp = os_utils_getenv("IPFS_PATH");
if (temp != NULL)
home_directory = temp;
} }
// get the full path // get the full path
int dir_len = strlen(home_directory) + 7; int dir_len = strlen(home_directory) + 7;

10
test/node/test_importer.h
View file

@ -46,13 +46,13 @@ int test_import_large_file() {
// write to ipfs // write to ipfs
struct Node* write_node; struct Node* write_node;
size_t bytes_written; size_t bytes_written;
if (ipfs_import_file("/tmp", fileName, &write_node, fs_repo, &bytes_written) == 0) { if (ipfs_import_file("/tmp", fileName, &write_node, fs_repo, &bytes_written, 1) == 0) {
ipfs_repo_fsrepo_free(fs_repo); ipfs_repo_fsrepo_free(fs_repo);
return 0; return 0;
} }
// cid should be the same each time // cid should be the same each time
unsigned char cid_test[10] = { 0x2c ,0x8e ,0x20 ,0x1b, 0xc7, 0xcc, 0x4d, 0x8f, 0x7e, 0x77 }; unsigned char cid_test[10] = { 0xc1 ,0x69 ,0x68 ,0x22, 0xfa, 0x47, 0x16, 0xe2, 0x41, 0xa1 };
/* /*
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
@ -195,13 +195,13 @@ int test_import_small_file() {
// write to ipfs // write to ipfs
struct Node* write_node; struct Node* write_node;
size_t bytes_written; size_t bytes_written;
if (ipfs_import_file("/tmp", fileName, &write_node, fs_repo, &bytes_written) == 0) { if (ipfs_import_file("/tmp", fileName, &write_node, fs_repo, &bytes_written, 1) == 0) {
ipfs_repo_fsrepo_free(fs_repo); ipfs_repo_fsrepo_free(fs_repo);
return 0; return 0;
} }
// cid should be the same each time // cid should be the same each time
unsigned char cid_test[10] = { 0x94, 0x4f, 0x39, 0xa0, 0x33, 0x5d, 0x7f, 0xf2, 0xcd, 0x66 }; unsigned char cid_test[10] = { 0x1e, 0xcf, 0x04, 0xce, 0x6a, 0xe8, 0xbf, 0xc0, 0xeb, 0xe4 };
/* /*
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
@ -227,7 +227,7 @@ int test_import_small_file() {
} }
// compare data // compare data
if (write_node->data_size != bytes_size || write_node->data_size != read_node->data_size) { if (write_node->data_size != bytes_size + 8 || write_node->data_size != read_node->data_size) {
printf("Data size of nodes are not equal or are incorrect. Should be %lu but are %lu\n", write_node->data_size, read_node->data_size); printf("Data size of nodes are not equal or are incorrect. Should be %lu but are %lu\n", write_node->data_size, read_node->data_size);
ipfs_repo_fsrepo_free(fs_repo); ipfs_repo_fsrepo_free(fs_repo);
ipfs_node_free(write_node); ipfs_node_free(write_node);

15
test/node/test_resolver.h
View file

@ -1,19 +1,24 @@
#include "ipfs/importer/resolver.h" #include "ipfs/importer/resolver.h"
#include "ipfs/os/utils.h"
int test_resolver_get() { int test_resolver_get() {
// clean out repository // clean out repository
drop_and_build_repository("/Users/JohnJones/.ipfs"); const char* ipfs_path = "/tmp/.ipfs";
os_utils_setenv("IPFS_PATH", ipfs_path, 1);
int argc = 3; drop_and_build_repository(ipfs_path);
int argc = 4;
char* argv[argc]; char* argv[argc];
argv[0] = "ipfs"; argv[0] = "ipfs";
argv[1] = "add"; argv[1] = "add";
argv[2] = "/Users/JohnJones/ipfstest"; argv[2] = "-r";
argv[3] = "/Users/JohnJones/ipfstest";
ipfs_import_files(argc, (char**)argv); ipfs_import_files(argc, (char**)argv);
struct FSRepo* fs_repo; struct FSRepo* fs_repo;
ipfs_repo_fsrepo_new("/Users/JohnJones/.ipfs", NULL, &fs_repo); ipfs_repo_fsrepo_new(ipfs_path, NULL, &fs_repo);
ipfs_repo_fsrepo_open(fs_repo); ipfs_repo_fsrepo_open(fs_repo);
// find something that is already in the repository // find something that is already in the repository
@ -25,7 +30,7 @@ int test_resolver_get() {
ipfs_node_free(result); ipfs_node_free(result);
// find something by path // find something by path
result = ipfs_resolver_get("/ipfs/QmWKtXwRg4oL2KaXhvJ3KyGjFE2PVKREwu7qb65V7ficui/hello_world.txt", NULL, fs_repo); result = ipfs_resolver_get("/ipfs/QmZBvycPAYScBoPEzm35zXHt6gYYV5t9PyWmr4sksLPNFS/hello_world.txt", NULL, fs_repo);
if (result == NULL) { if (result == NULL) {
return 0; return 0;
} }

6
test/repo/test_repo_config.h
View file

@ -1,6 +1,6 @@
#ifndef test_repo_config_h #ifndef test_repo_config_h
#define test_repo_config_h #define test_repo_config_h
#include <sys/stat.h>
#include "ipfs/repo/config/config.h" #include "ipfs/repo/config/config.h"
#include "ipfs/repo/fsrepo/fs_repo.h" #include "ipfs/repo/fsrepo/fs_repo.h"
#include "ipfs/os/utils.h" #include "ipfs/os/utils.h"
@ -63,6 +63,10 @@ int test_repo_config_init() {
* test the writing of the config file * test the writing of the config file
*/ */
int test_repo_config_write() { int test_repo_config_write() {
// make sure the directory is there
if (!os_utils_file_exists("/tmp/.ipfs")) {
mkdir("/tmp/.ipfs", S_IRWXU);
}
// first delete the existing one // first delete the existing one
unlink("/tmp/.ipfs/config"); unlink("/tmp/.ipfs/config");

5
test/storage/test_unixfs.h
View file

@ -65,8 +65,7 @@ int test_unixfs_encode_smallfile() {
0x74, 0x68, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x69, 0x6e, 0x20,
0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x48, 0x65, 0x6c, 0x6c, 0x6f,
0x57, 0x65, 0x72, 0x6c, 0x64, 0x57, 0x65, 0x72, 0x6c, 0x64,
0x2e, 0x74, 0x78, 0x74, 0x0a, 0x2e, 0x74, 0x78, 0x74, 0x0a
0x18, 0x23
}; };
unixfs->bytes = (unsigned char*)malloc(35); unixfs->bytes = (unsigned char*)malloc(35);
@ -81,7 +80,7 @@ int test_unixfs_encode_smallfile() {
int retVal = 1; int retVal = 1;
if (bytes_written != 41) { if (bytes_written != 39) {
printf("Length should be %lu, but is %lu\n", 41LU, bytes_written); printf("Length should be %lu, but is %lu\n", 41LU, bytes_written);
retVal = 0; retVal = 0;
} }