diff --git a/importer/importer.c b/importer/importer.c index 5861afd..02daf4f 100644 --- a/importer/importer.c +++ b/importer/importer.c @@ -50,7 +50,7 @@ int ipfs_importer_add_filesize_to_data_section(struct Node* node, size_t bytes_r * @param node the node to add to * @returns number of bytes read */ -size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs_repo, size_t* total_size) { +size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs_repo, size_t* total_size, size_t* bytes_written) { unsigned char buffer[MAX_DATA_SIZE]; size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file); @@ -75,8 +75,8 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs return 0; } unsigned char protobuf[protobuf_size]; - size_t bytes_written = 0; - if (ipfs_unixfs_protobuf_encode(new_unixfs, protobuf, protobuf_size, &bytes_written) == 0) { + *bytes_written = 0; + if (ipfs_unixfs_protobuf_encode(new_unixfs, protobuf, protobuf_size, bytes_written) == 0) { ipfs_unixfs_free(new_unixfs); return 0; } @@ -84,10 +84,12 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs // we're done with the UnixFS object ipfs_unixfs_free(new_unixfs); + size_t size_of_node = 0; + // if there is more to read, create a new node. if (bytes_read == MAX_DATA_SIZE) { // create a new node - if (ipfs_node_new_from_data(protobuf, bytes_written, &new_node) == 0) { + if (ipfs_node_new_from_data(protobuf, *bytes_written, &new_node) == 0) { return 0; } // persist @@ -111,19 +113,20 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs } ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read); ipfs_node_free(new_node); + *bytes_written = size_of_node; + size_of_node = 0; } else { // if there are no existing links, put what we pulled from the file into parent_node // otherwise, add it as a link if (parent_node->head_link == NULL) { - ipfs_node_set_data(parent_node, protobuf, bytes_written); + ipfs_node_set_data(parent_node, protobuf, *bytes_written); } else { // there are existing links. put the data in a new node, save it, then put the link in parent_node // create a new node - if (ipfs_node_new_from_data(protobuf, bytes_written, &new_node) == 0) { + if (ipfs_node_new_from_data(protobuf, *bytes_written, &new_node) == 0) { return 0; } // persist - size_t size_of_node = 0; if (ipfs_merkledag_add(new_node, fs_repo, &size_of_node) == 0) { ipfs_node_free(new_node); return 0; @@ -145,7 +148,8 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs ipfs_node_free(new_node); } // persist the main node - ipfs_merkledag_add(parent_node, fs_repo, &bytes_written); + ipfs_merkledag_add(parent_node, fs_repo, bytes_written); + *bytes_written += size_of_node; } // add to parent vs add as link return bytes_read; @@ -178,7 +182,7 @@ int ipfs_import_print_node_results(const struct Node* node, const char* file_nam * @param parent_node the root node (has links to others in case this is a large file and is split) * @returns true(1) on success */ -int ipfs_import_file(const char* fileName, struct Node** parent_node, struct FSRepo* fs_repo) { +int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** parent_node, struct FSRepo* fs_repo, size_t* bytes_written) { /** * NOTE: When this function completes, parent_node will be either: * 1) the complete file, in the case of a small file (<256k-ish) @@ -198,22 +202,26 @@ int ipfs_import_file(const char* fileName, struct Node** parent_node, struct FSR struct FileList* next = first; while (next != NULL) { // process each file. NOTE: could be an embedded directory + *bytes_written = 0; struct Node* file_node; size_t filename_len = strlen(fileName) + strlen(next->file_name) + 2; char full_file_name[filename_len]; os_utils_filepath_join(fileName, next->file_name, full_file_name, filename_len); - if (ipfs_import_file(full_file_name, &file_node, fs_repo) == 0) { + if (ipfs_import_file(root_dir, full_file_name, &file_node, fs_repo, bytes_written) == 0) { ipfs_node_free(*parent_node); os_utils_free_file_list(first); return 0; } // TODO: probably need to display what was imported - ipfs_import_print_node_results(file_node, next->file_name); + int len = strlen(next->file_name) + strlen(root_dir) + 2; + char full_path[len]; + os_utils_filepath_join(root_dir, next->file_name, full_path, len); + ipfs_import_print_node_results(file_node, full_path); // TODO: Determine what needs to be done if this file_node is a file, a split file, or a directory // Create link from file_node struct NodeLink* file_node_link; ipfs_node_link_create(next->file_name, file_node->hash, file_node->hash_size, &file_node_link); - file_node_link->t_size = file_node->data_size; + file_node_link->t_size = *bytes_written; // add file_node as link to parent_node ipfs_node_add_link(*parent_node, file_node_link); // clean up file_node @@ -233,7 +241,9 @@ int ipfs_import_file(const char* fileName, struct Node** parent_node, struct FSR // add all nodes (will be called multiple times for large files) while ( bytes_read == MAX_DATA_SIZE) { - bytes_read = ipfs_import_chunk(file, *parent_node, fs_repo, &total_size); + size_t written = 0; + bytes_read = ipfs_import_chunk(file, *parent_node, fs_repo, &total_size, &written); + *bytes_written += written; } fclose(file); } @@ -290,9 +300,13 @@ int ipfs_import_files(int argc, char** argv) { struct FileList* current = first; while (current != NULL) { struct Node* directory_entry = NULL; - retVal = ipfs_import_file(current->file_name, &directory_entry, fs_repo); + char* path; + char* filename; + os_utils_split_filename(current->file_name, &path, &filename); + size_t bytes_written = 0; + retVal = ipfs_import_file(filename, current->file_name, &directory_entry, fs_repo, &bytes_written); - ipfs_import_print_node_results(directory_entry, current->file_name); + ipfs_import_print_node_results(directory_entry, filename); // cleanup ipfs_node_free(directory_entry); current = current->next; diff --git a/include/ipfs/importer/importer.h b/include/ipfs/importer/importer.h index 83efadb..3b4b0e9 100644 --- a/include/ipfs/importer/importer.h +++ b/include/ipfs/importer/importer.h @@ -10,7 +10,7 @@ * @param node the root node (could have links to others) * @returns true(1) on success */ -int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo); +int ipfs_import_file(const char* root, const char* fileName, struct Node** node, struct FSRepo* fs_repo, size_t* bytes_written); /** * called from the command line diff --git a/include/ipfs/os/utils.h b/include/ipfs/os/utils.h index a1b26d8..2753ced 100644 --- a/include/ipfs/os/utils.h +++ b/include/ipfs/os/utils.h @@ -16,10 +16,30 @@ struct FileList { struct FileList* next; }; +/** + * Builds a list of files within a directory + * @param path the path to examine + * @returns a FileList struct of the first file + */ struct FileList* os_utils_list_directory(const char* path); -// frees memory used by creating a FileList linked list + +/** + * Cleans up memory used by a FileList struct + * @param first the struct to free + * @returns true(1) + */ int os_utils_free_file_list(struct FileList* first); +/** + * Split the filename from the path + * @param in the full path and filename + * @param path only the path part + * @param filename only the file name + * @returns true(1) + */ +int os_utils_split_filename(const char* in, char** path, char** filename); + + /** * get an environment varible from the os * @param variable the variable to look for diff --git a/os/Makefile b/os/Makefile index 66eaabe..2ab0663 100644 --- a/os/Makefile +++ b/os/Makefile @@ -1,5 +1,11 @@ CC = gcc CFLAGS = -O0 -I../include -I../../c-libp2p/include -Wall + +ifdef DEBUG +CFLAGS += -g3 +endif + + LFLAGS = DEPS = OBJS = utils.o diff --git a/os/utils.c b/os/utils.c index 5bae8fb..93c5f80 100644 --- a/os/utils.c +++ b/os/utils.c @@ -64,6 +64,19 @@ int os_utils_is_directory(const char* file_name) { return S_ISDIR(path_stat.st_mode) != 0; } +int os_utils_split_filename(const char* in, char** path, char** filename) { + int len = strlen(in); + char* pos = strrchr(in, '/'); + pos++; + *path = (char*)malloc((pos - in) + 1); + *filename = (char*)malloc(len - (pos-in) + 1); + strncpy(*path, in, pos-in-1); + (*path)[pos-in-1] = 0; + strcpy(*filename, pos); + (*filename)[len - (pos-in)] = 0; + return 1; +} + struct FileList* os_utils_list_directory(const char* path) { DIR* dp; struct dirent *ep; diff --git a/test/node/test_importer.h b/test/node/test_importer.h index 285e4bd..5a5f3f2 100644 --- a/test/node/test_importer.h +++ b/test/node/test_importer.h @@ -45,7 +45,8 @@ int test_import_large_file() { // write to ipfs struct Node* write_node; - if (ipfs_import_file(fileName, &write_node, fs_repo) == 0) { + size_t bytes_written; + if (ipfs_import_file("/tmp", fileName, &write_node, fs_repo, &bytes_written) == 0) { ipfs_repo_fsrepo_free(fs_repo); return 0; } @@ -193,7 +194,8 @@ int test_import_small_file() { // write to ipfs struct Node* write_node; - if (ipfs_import_file(fileName, &write_node, fs_repo) == 0) { + size_t bytes_written; + if (ipfs_import_file("/tmp", fileName, &write_node, fs_repo, &bytes_written) == 0) { ipfs_repo_fsrepo_free(fs_repo); return 0; }