Directory hashes match

This commit is contained in:
jmjatlanta 2016-12-29 04:42:01 -05:00
parent fa3dd77e96
commit 396dfc6abc
6 changed files with 74 additions and 19 deletions

View file

@ -50,7 +50,7 @@ int ipfs_importer_add_filesize_to_data_section(struct Node* node, size_t bytes_r
* @param node the node to add to * @param node the node to add to
* @returns number of bytes read * @returns number of bytes read
*/ */
size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs_repo, size_t* total_size) { size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs_repo, size_t* total_size, size_t* bytes_written) {
unsigned char buffer[MAX_DATA_SIZE]; unsigned char buffer[MAX_DATA_SIZE];
size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file); size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file);
@ -75,8 +75,8 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
return 0; return 0;
} }
unsigned char protobuf[protobuf_size]; unsigned char protobuf[protobuf_size];
size_t bytes_written = 0; *bytes_written = 0;
if (ipfs_unixfs_protobuf_encode(new_unixfs, protobuf, protobuf_size, &bytes_written) == 0) { if (ipfs_unixfs_protobuf_encode(new_unixfs, protobuf, protobuf_size, bytes_written) == 0) {
ipfs_unixfs_free(new_unixfs); ipfs_unixfs_free(new_unixfs);
return 0; return 0;
} }
@ -84,10 +84,12 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
// we're done with the UnixFS object // we're done with the UnixFS object
ipfs_unixfs_free(new_unixfs); ipfs_unixfs_free(new_unixfs);
size_t size_of_node = 0;
// if there is more to read, create a new node. // if there is more to read, create a new node.
if (bytes_read == MAX_DATA_SIZE) { if (bytes_read == MAX_DATA_SIZE) {
// create a new node // create a new node
if (ipfs_node_new_from_data(protobuf, bytes_written, &new_node) == 0) { if (ipfs_node_new_from_data(protobuf, *bytes_written, &new_node) == 0) {
return 0; return 0;
} }
// persist // persist
@ -111,19 +113,20 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
} }
ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read); ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read);
ipfs_node_free(new_node); ipfs_node_free(new_node);
*bytes_written = size_of_node;
size_of_node = 0;
} else { } else {
// if there are no existing links, put what we pulled from the file into parent_node // if there are no existing links, put what we pulled from the file into parent_node
// otherwise, add it as a link // otherwise, add it as a link
if (parent_node->head_link == NULL) { if (parent_node->head_link == NULL) {
ipfs_node_set_data(parent_node, protobuf, bytes_written); ipfs_node_set_data(parent_node, protobuf, *bytes_written);
} else { } else {
// there are existing links. put the data in a new node, save it, then put the link in parent_node // there are existing links. put the data in a new node, save it, then put the link in parent_node
// create a new node // create a new node
if (ipfs_node_new_from_data(protobuf, bytes_written, &new_node) == 0) { if (ipfs_node_new_from_data(protobuf, *bytes_written, &new_node) == 0) {
return 0; return 0;
} }
// persist // persist
size_t size_of_node = 0;
if (ipfs_merkledag_add(new_node, fs_repo, &size_of_node) == 0) { if (ipfs_merkledag_add(new_node, fs_repo, &size_of_node) == 0) {
ipfs_node_free(new_node); ipfs_node_free(new_node);
return 0; return 0;
@ -145,7 +148,8 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
ipfs_node_free(new_node); ipfs_node_free(new_node);
} }
// persist the main node // persist the main node
ipfs_merkledag_add(parent_node, fs_repo, &bytes_written); ipfs_merkledag_add(parent_node, fs_repo, bytes_written);
*bytes_written += size_of_node;
} // add to parent vs add as link } // add to parent vs add as link
return bytes_read; return bytes_read;
@ -178,7 +182,7 @@ int ipfs_import_print_node_results(const struct Node* node, const char* file_nam
* @param parent_node the root node (has links to others in case this is a large file and is split) * @param parent_node the root node (has links to others in case this is a large file and is split)
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_import_file(const char* fileName, struct Node** parent_node, struct FSRepo* fs_repo) { int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** parent_node, struct FSRepo* fs_repo, size_t* bytes_written) {
/** /**
* NOTE: When this function completes, parent_node will be either: * NOTE: When this function completes, parent_node will be either:
* 1) the complete file, in the case of a small file (<256k-ish) * 1) the complete file, in the case of a small file (<256k-ish)
@ -198,22 +202,26 @@ int ipfs_import_file(const char* fileName, struct Node** parent_node, struct FSR
struct FileList* next = first; struct FileList* next = first;
while (next != NULL) { while (next != NULL) {
// process each file. NOTE: could be an embedded directory // process each file. NOTE: could be an embedded directory
*bytes_written = 0;
struct Node* file_node; struct Node* file_node;
size_t filename_len = strlen(fileName) + strlen(next->file_name) + 2; size_t filename_len = strlen(fileName) + strlen(next->file_name) + 2;
char full_file_name[filename_len]; char full_file_name[filename_len];
os_utils_filepath_join(fileName, next->file_name, full_file_name, filename_len); os_utils_filepath_join(fileName, next->file_name, full_file_name, filename_len);
if (ipfs_import_file(full_file_name, &file_node, fs_repo) == 0) { if (ipfs_import_file(root_dir, full_file_name, &file_node, fs_repo, bytes_written) == 0) {
ipfs_node_free(*parent_node); ipfs_node_free(*parent_node);
os_utils_free_file_list(first); os_utils_free_file_list(first);
return 0; return 0;
} }
// TODO: probably need to display what was imported // TODO: probably need to display what was imported
ipfs_import_print_node_results(file_node, next->file_name); int len = strlen(next->file_name) + strlen(root_dir) + 2;
char full_path[len];
os_utils_filepath_join(root_dir, next->file_name, full_path, len);
ipfs_import_print_node_results(file_node, full_path);
// TODO: Determine what needs to be done if this file_node is a file, a split file, or a directory // TODO: Determine what needs to be done if this file_node is a file, a split file, or a directory
// Create link from file_node // Create link from file_node
struct NodeLink* file_node_link; struct NodeLink* file_node_link;
ipfs_node_link_create(next->file_name, file_node->hash, file_node->hash_size, &file_node_link); ipfs_node_link_create(next->file_name, file_node->hash, file_node->hash_size, &file_node_link);
file_node_link->t_size = file_node->data_size; file_node_link->t_size = *bytes_written;
// add file_node as link to parent_node // add file_node as link to parent_node
ipfs_node_add_link(*parent_node, file_node_link); ipfs_node_add_link(*parent_node, file_node_link);
// clean up file_node // clean up file_node
@ -233,7 +241,9 @@ int ipfs_import_file(const char* fileName, struct Node** parent_node, struct FSR
// add all nodes (will be called multiple times for large files) // add all nodes (will be called multiple times for large files)
while ( bytes_read == MAX_DATA_SIZE) { while ( bytes_read == MAX_DATA_SIZE) {
bytes_read = ipfs_import_chunk(file, *parent_node, fs_repo, &total_size); size_t written = 0;
bytes_read = ipfs_import_chunk(file, *parent_node, fs_repo, &total_size, &written);
*bytes_written += written;
} }
fclose(file); fclose(file);
} }
@ -290,9 +300,13 @@ int ipfs_import_files(int argc, char** argv) {
struct FileList* current = first; struct FileList* current = first;
while (current != NULL) { while (current != NULL) {
struct Node* directory_entry = NULL; struct Node* directory_entry = NULL;
retVal = ipfs_import_file(current->file_name, &directory_entry, fs_repo); char* path;
char* filename;
os_utils_split_filename(current->file_name, &path, &filename);
size_t bytes_written = 0;
retVal = ipfs_import_file(filename, current->file_name, &directory_entry, fs_repo, &bytes_written);
ipfs_import_print_node_results(directory_entry, current->file_name); ipfs_import_print_node_results(directory_entry, filename);
// cleanup // cleanup
ipfs_node_free(directory_entry); ipfs_node_free(directory_entry);
current = current->next; current = current->next;

View file

@ -10,7 +10,7 @@
* @param node the root node (could have links to others) * @param node the root node (could have links to others)
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo); int ipfs_import_file(const char* root, const char* fileName, struct Node** node, struct FSRepo* fs_repo, size_t* bytes_written);
/** /**
* called from the command line * called from the command line

View file

@ -16,10 +16,30 @@ struct FileList {
struct FileList* next; struct FileList* next;
}; };
/**
* Builds a list of files within a directory
* @param path the path to examine
* @returns a FileList struct of the first file
*/
struct FileList* os_utils_list_directory(const char* path); struct FileList* os_utils_list_directory(const char* path);
// frees memory used by creating a FileList linked list
/**
* Cleans up memory used by a FileList struct
* @param first the struct to free
* @returns true(1)
*/
int os_utils_free_file_list(struct FileList* first); int os_utils_free_file_list(struct FileList* first);
/**
* Split the filename from the path
* @param in the full path and filename
* @param path only the path part
* @param filename only the file name
* @returns true(1)
*/
int os_utils_split_filename(const char* in, char** path, char** filename);
/** /**
* get an environment varible from the os * get an environment varible from the os
* @param variable the variable to look for * @param variable the variable to look for

View file

@ -1,5 +1,11 @@
CC = gcc CC = gcc
CFLAGS = -O0 -I../include -I../../c-libp2p/include -Wall CFLAGS = -O0 -I../include -I../../c-libp2p/include -Wall
ifdef DEBUG
CFLAGS += -g3
endif
LFLAGS = LFLAGS =
DEPS = DEPS =
OBJS = utils.o OBJS = utils.o

View file

@ -64,6 +64,19 @@ int os_utils_is_directory(const char* file_name) {
return S_ISDIR(path_stat.st_mode) != 0; return S_ISDIR(path_stat.st_mode) != 0;
} }
int os_utils_split_filename(const char* in, char** path, char** filename) {
int len = strlen(in);
char* pos = strrchr(in, '/');
pos++;
*path = (char*)malloc((pos - in) + 1);
*filename = (char*)malloc(len - (pos-in) + 1);
strncpy(*path, in, pos-in-1);
(*path)[pos-in-1] = 0;
strcpy(*filename, pos);
(*filename)[len - (pos-in)] = 0;
return 1;
}
struct FileList* os_utils_list_directory(const char* path) { struct FileList* os_utils_list_directory(const char* path) {
DIR* dp; DIR* dp;
struct dirent *ep; struct dirent *ep;

View file

@ -45,7 +45,8 @@ int test_import_large_file() {
// write to ipfs // write to ipfs
struct Node* write_node; struct Node* write_node;
if (ipfs_import_file(fileName, &write_node, fs_repo) == 0) { size_t bytes_written;
if (ipfs_import_file("/tmp", fileName, &write_node, fs_repo, &bytes_written) == 0) {
ipfs_repo_fsrepo_free(fs_repo); ipfs_repo_fsrepo_free(fs_repo);
return 0; return 0;
} }
@ -193,7 +194,8 @@ int test_import_small_file() {
// write to ipfs // write to ipfs
struct Node* write_node; struct Node* write_node;
if (ipfs_import_file(fileName, &write_node, fs_repo) == 0) { size_t bytes_written;
if (ipfs_import_file("/tmp", fileName, &write_node, fs_repo, &bytes_written) == 0) {
ipfs_repo_fsrepo_free(fs_repo); ipfs_repo_fsrepo_free(fs_repo);
return 0; return 0;
} }