Major changes to support large file transfer

yamux
John Jones 2017-04-20 17:56:03 -05:00
parent a2a08156a7
commit 03696dd6e7
40 changed files with 875 additions and 427 deletions

View File

@ -235,7 +235,7 @@ int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, st
* @param bytes_written the number of bytes written to the blockstore
* @returns true(1) on success
*/
int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written) {
int ipfs_blockstore_put_node(const struct HashtableNode* node, const struct FSRepo* fs_repo, size_t* bytes_written) {
// from blockstore.go line 118
int retVal = 0;
@ -249,9 +249,9 @@ int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_re
//TODO: put this in subdirectories
// turn the block into a binary array
size_t protobuf_len = ipfs_node_protobuf_encode_size(node);
size_t protobuf_len = ipfs_hashtable_node_protobuf_encode_size(node);
unsigned char protobuf[protobuf_len];
retVal = ipfs_node_protobuf_encode(node, protobuf, protobuf_len, &protobuf_len);
retVal = ipfs_hashtable_node_protobuf_encode(node, protobuf, protobuf_len, &protobuf_len);
if (retVal == 0) {
free(key);
return 0;
@ -286,7 +286,7 @@ int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_re
* @param fs_repo where to look for the data
* @returns true(1) on success
*/
int ipfs_blockstore_get_node(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo) {
int ipfs_blockstore_get_node(const unsigned char* hash, size_t hash_length, struct HashtableNode** node, const struct FSRepo* fs_repo) {
// get datastore key, which is a base32 key of the multihash
unsigned char* key = ipfs_blockstore_hash_to_base32(hash, hash_length);
@ -299,7 +299,7 @@ int ipfs_blockstore_get_node(const unsigned char* hash, size_t hash_length, stru
size_t bytes_read = fread(buffer, 1, file_size, file);
fclose(file);
int retVal = ipfs_node_protobuf_decode(buffer, bytes_read, node);
int retVal = ipfs_hashtable_node_protobuf_decode(buffer, bytes_read, node);
free(key);
free(filename);

View File

@ -7,7 +7,7 @@ endif
LFLAGS =
DEPS = builder.h ipfs_node.h
OBJS = builder.o daemon.o null.o ping.o bootstrap.o
OBJS = builder.o daemon.o null.o ping.o bootstrap.o ipfs_node.o
%.o: %.c $(DEPS)
$(CC) -c -o $@ $< $(CFLAGS)

View File

@ -47,7 +47,7 @@ void ipfs_bootstrap_announce_files(struct IpfsNode* local_node) {
enum DatastoreCursorOp op = CURSOR_FIRST;
while (db->datastore_cursor_get(&key, &key_size, NULL, 0, op, db)) {
libp2p_logger_debug("bootstrap", "Announcing a file to the world.\n");
local_node->routing->Provide(local_node->routing, (char*)key, key_size);
local_node->routing->Provide(local_node->routing, key, key_size);
op = CURSOR_NEXT;
free(key);
}
@ -66,7 +66,6 @@ void ipfs_bootstrap_announce_files(struct IpfsNode* local_node) {
* @returns nothing useful
*/
void *ipfs_bootstrap_routing(void* param) {
libp2p_logger_add_class("bootstrap");
struct IpfsNode* local_node = (struct IpfsNode*)param;
local_node->routing = ipfs_routing_new_online(local_node, &local_node->identity->private_key, NULL);
local_node->routing->Bootstrap(local_node->routing);

View File

@ -20,29 +20,15 @@ int ipfs_daemon_start(char* repo_path) {
libp2p_logger_info("daemon", "Initializing daemon...\n");
// read the configuration
struct FSRepo* fs_repo = NULL;
if (!ipfs_repo_fsrepo_new(repo_path, NULL, &fs_repo))
goto exit;
// open the repository and read the file
if (!ipfs_repo_fsrepo_open(fs_repo)) {
goto exit;
}
// create a new IpfsNode
struct IpfsNode local_node;
local_node.mode = MODE_ONLINE;
local_node.peerstore = libp2p_peerstore_new();
local_node.providerstore = libp2p_providerstore_new();
local_node.repo = fs_repo;
local_node.identity = fs_repo->config->identity;
struct IpfsNode* local_node = NULL;
if (!ipfs_node_online_new(repo_path, &local_node))
goto exit;
// Set null router param
ma = multiaddress_new_from_string(fs_repo->config->addresses->swarm_head->item);
ma = multiaddress_new_from_string(local_node->repo->config->addresses->swarm_head->item);
listen_param.port = multiaddress_get_ip_port(ma);
listen_param.ipv4 = 0; // ip 0.0.0.0, all interfaces
listen_param.local_node = &local_node;
listen_param.local_node = local_node;
// Create pthread for swarm listener.
if (pthread_create(&work_pths[count_pths++], NULL, ipfs_null_listen, &listen_param)) {
@ -50,7 +36,7 @@ int ipfs_daemon_start(char* repo_path) {
goto exit;
}
ipfs_bootstrap_routing(&local_node);
//ipfs_bootstrap_routing(local_node);
libp2p_logger_info("daemon", "Daemon is ready\n");
@ -64,18 +50,12 @@ int ipfs_daemon_start(char* repo_path) {
retVal = 1;
exit:
fprintf(stderr, "Cleaning up daemon processes\n");
libp2p_logger_debug("daemon", "Cleaning up daemon processes\n");
// clean up
if (fs_repo != NULL)
ipfs_repo_fsrepo_free(fs_repo);
if (local_node.peerstore != NULL)
libp2p_peerstore_free(local_node.peerstore);
if (local_node.providerstore != NULL)
libp2p_providerstore_free(local_node.providerstore);
if (ma != NULL)
multiaddress_free(ma);
if (local_node.routing != NULL) {
ipfs_routing_online_free(local_node.routing);
if (local_node != NULL) {
ipfs_node_free(local_node);
}
return retVal;
@ -94,10 +74,5 @@ int ipfs_daemon (int argc, char **argv)
return 0;
}
libp2p_logger_add_class("peerstore");
libp2p_logger_add_class("providerstore");
libp2p_logger_add_class("daemon");
libp2p_logger_add_class("online");
return ipfs_daemon_start(repo_path);
}

61
core/ipfs_node.c Normal file
View File

@ -0,0 +1,61 @@
#include <stdlib.h>
#include "ipfs/core/ipfs_node.h"
/***
* build an online IpfsNode
* @param repo_path where the IPFS repository directory is
* @param node the completed IpfsNode struct
* @returns true(1) on success
*/
int ipfs_node_online_new(const char* repo_path, struct IpfsNode** node) {
struct FSRepo* fs_repo = NULL;
*node = (struct IpfsNode*)malloc(sizeof(struct IpfsNode));
if(*node == NULL)
return 0;
struct IpfsNode* local_node = *node;
// build the struct
if (!ipfs_repo_fsrepo_new(repo_path, NULL, &fs_repo)) {
free(local_node);
return 0;
}
// open the repo
if (!ipfs_repo_fsrepo_open(fs_repo)) {
free(local_node);
return 0;
}
// fill in the node
local_node->repo = fs_repo;
local_node->identity = fs_repo->config->identity;
local_node->peerstore = libp2p_peerstore_new();
local_node->providerstore = libp2p_providerstore_new();
local_node->mode = MODE_OFFLINE;
local_node->routing = ipfs_routing_new_online(local_node, &fs_repo->config->identity->private_key, NULL);
return 1;
}
/***
* Free resources from the creation of an IpfsNode
* @param node the node to free
* @returns true(1)
*/
int ipfs_node_free(struct IpfsNode* node) {
if (node != NULL) {
if (node->providerstore != NULL)
libp2p_providerstore_free(node->providerstore);
if (node->peerstore != NULL)
libp2p_peerstore_free(node->peerstore);
if (node->repo != NULL)
ipfs_repo_fsrepo_free(node->repo);
if (node->mode == MODE_ONLINE) {
ipfs_routing_online_free(node->routing);
}
free(node);
}
return 1;
}

View File

@ -54,6 +54,7 @@ void ipfs_null_connection (void *ptr)
session.insecure_stream = libp2p_net_multistream_stream_new(connection_param->file_descriptor, connection_param->ip, connection_param->port);
session.default_stream = session.insecure_stream;
session.datastore = connection_param->local_node->repo->config->datastore;
session.filestore = connection_param->local_node->repo->config->filestore;
libp2p_logger_log("null", LOGLEVEL_INFO, "Connection %d, count %d\n", connection_param->file_descriptor, *(connection_param->count));
@ -100,14 +101,14 @@ void ipfs_null_connection (void *ptr)
}
else {
// try to get the Node
struct Node* node = NULL;
struct HashtableNode* node = NULL;
if (!ipfs_merkledag_get(hash, hash_length, &node, connection_param->local_node->repo)) {
_continue = 0;
continue;
}
size_t results_size = ipfs_node_protobuf_encode_size(node);
size_t results_size = ipfs_hashtable_node_protobuf_encode_size(node);
unsigned char results[results_size];
if (!ipfs_node_protobuf_encode(node, results, results_size, &results_size)) {
if (!ipfs_hashtable_node_protobuf_encode(node, results, results_size, &results_size)) {
_continue = 0;
continue;
}

View File

@ -6,18 +6,62 @@
#include "ipfs/merkledag/node.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
#include "ipfs/repo/init.h"
#include "ipfs/core/ipfs_node.h"
#include "libp2p/utils/logger.h"
/**
* pull objects from ipfs
*/
/***
* Helper method to retrieve a protobuf'd Node from the router
* @param local_node the context
* @param hash the hash to retrieve
* @param hash_size the length of the hash
* @param result a place to store the Node
* @returns true(1) on success, otherwise false(0)
*/
int ipfs_exporter_get_node(struct IpfsNode* local_node, const unsigned char* hash, const size_t hash_size,
struct HashtableNode** result) {
unsigned char *buffer = NULL;
size_t buffer_size = 0;
int retVal = 0;
struct Libp2pMessage* msg = NULL;
if (local_node->routing->GetValue(local_node->routing, hash, hash_size, (void**)&buffer, &buffer_size)) {
libp2p_logger_debug("exporter", "get_node got a value. Converting it to a HashtableNode\n");
// unprotobuf
if (ipfs_hashtable_node_protobuf_decode(buffer, buffer_size, result)) {
libp2p_logger_debug("exporter", "Conversion to HashtableNode successful\n");
}
} else {
libp2p_logger_debug("exporter", "get_node got no value. Returning false.\n");
goto exit;
}
// copy in the hash
(*result)->hash_size = hash_size;
(*result)->hash = malloc(hash_size);
memcpy((*result)->hash, hash, hash_size);
retVal = 1;
exit:
if (buffer != NULL)
free(buffer);
if (msg != NULL)
libp2p_message_free(msg);
return retVal;
}
/***
* Get a file by its hash, and write the data to a filestream
* @param hash the base58 multihash of the cid
* @param file_descriptor where to write
* @param fs_repo the repo
* @param local_node the context
*/
int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor, const struct FSRepo* fs_repo) {
int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor, struct IpfsNode* local_node) {
// convert hash to cid
struct Cid* cid = NULL;
if ( ipfs_cid_decode_hash_from_base58(hash, strlen((char*)hash), &cid) == 0) {
@ -25,8 +69,8 @@ int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor
}
// find block
struct Node* read_node = NULL;
if (ipfs_merkledag_get(cid->hash, cid->hash_length, &read_node, fs_repo) == 0) {
struct HashtableNode* read_node = NULL;
if (!ipfs_exporter_get_node(local_node, cid->hash, cid->hash_length, &read_node)) {
ipfs_cid_free(cid);
return 0;
}
@ -41,31 +85,31 @@ int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor
size_t bytes_written = fwrite(unix_fs->bytes, 1, unix_fs->bytes_size, file_descriptor);
if (bytes_written != unix_fs->bytes_size) {
fclose(file_descriptor);
ipfs_node_free(read_node);
ipfs_hashtable_node_free(read_node);
ipfs_unixfs_free(unix_fs);
return 0;
}
ipfs_unixfs_free(unix_fs);
} else {
struct NodeLink* link = read_node->head_link;
struct Node* link_node = NULL;
struct HashtableNode* link_node = NULL;
while (link != NULL) {
if ( ipfs_merkledag_get(link->hash, link->hash_size, &link_node, fs_repo) == 0) {
if ( !ipfs_exporter_get_node(local_node, link->hash, link->hash_size, &link_node)) {
fclose(file_descriptor);
ipfs_node_free(read_node);
ipfs_hashtable_node_free(read_node);
return 0;
}
struct UnixFS* unix_fs;
ipfs_unixfs_protobuf_decode(link_node->data, link_node->data_size, &unix_fs);
size_t bytes_written = fwrite(unix_fs->bytes, 1, unix_fs->bytes_size, file_descriptor);
if (bytes_written != unix_fs->bytes_size) {
ipfs_node_free(link_node);
ipfs_hashtable_node_free(link_node);
fclose(file_descriptor);
ipfs_node_free(read_node);
ipfs_hashtable_node_free(read_node);
ipfs_unixfs_free(unix_fs);
return 0;
}
ipfs_node_free(link_node);
ipfs_hashtable_node_free(link_node);
ipfs_unixfs_free(unix_fs);
link = link->next;
}
@ -73,7 +117,7 @@ int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor
fclose(file_descriptor);
if (read_node != NULL)
ipfs_node_free(read_node);
ipfs_hashtable_node_free(read_node);
return 1;
}
@ -85,13 +129,13 @@ int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor
* @param file_name the file name to write to
* @returns true(1) on success
*/
int ipfs_exporter_to_file(const unsigned char* hash, const char* file_name, const struct FSRepo* fs_repo) {
int ipfs_exporter_to_file(const unsigned char* hash, const char* file_name, struct IpfsNode *local_node) {
// process blocks
FILE* file = fopen(file_name, "wb");
if (file == NULL) {
return 0;
}
return ipfs_exporter_to_filestream(hash, file, fs_repo);
return ipfs_exporter_to_filestream(hash, file, local_node);
}
/**
@ -100,7 +144,7 @@ int ipfs_exporter_to_file(const unsigned char* hash, const char* file_name, cons
* @param file_name the file name to write to
* @returns true(1) on success
*/
int ipfs_exporter_to_console(const unsigned char* hash, const struct FSRepo* fs_repo) {
int ipfs_exporter_to_console(const unsigned char* hash, struct IpfsNode *local_node) {
// convert hash to cid
struct Cid* cid = NULL;
if ( ipfs_cid_decode_hash_from_base58(hash, strlen((char*)hash), &cid) == 0) {
@ -108,8 +152,8 @@ int ipfs_exporter_to_console(const unsigned char* hash, const struct FSRepo* fs_
}
// find block
struct Node* read_node = NULL;
if (ipfs_merkledag_get(cid->hash, cid->hash_length, &read_node, fs_repo) == 0) {
struct HashtableNode* read_node = NULL;
if (!ipfs_exporter_get_node(local_node, cid->hash, cid->hash_length, &read_node)) {
ipfs_cid_free(cid);
return 0;
}
@ -133,7 +177,7 @@ int ipfs_exporter_to_console(const unsigned char* hash, const struct FSRepo* fs_
printf("\"}\n");
if (read_node != NULL)
ipfs_node_free(read_node);
ipfs_hashtable_node_free(read_node);
return 1;
}
@ -147,7 +191,6 @@ int ipfs_exporter_to_console(const unsigned char* hash, const struct FSRepo* fs_
* @returns true(1) on success
*/
int ipfs_exporter_object_get(int argc, char** argv) {
struct FSRepo* fs_repo = NULL;
char* repo_path = NULL;
if (!ipfs_repo_get_directory(argc, argv, &repo_path)) {
@ -155,22 +198,19 @@ int ipfs_exporter_object_get(int argc, char** argv) {
return 0;
}
// build the struct
int retVal = ipfs_repo_fsrepo_new(repo_path, NULL, &fs_repo);
if (retVal == 0) {
struct IpfsNode* local_node = NULL;
if (!ipfs_node_online_new(repo_path, &local_node))
return 0;
}
// open the repo
retVal = ipfs_repo_fsrepo_open(fs_repo);
if (retVal == 0) {
return 0;
}
// find hash
retVal = ipfs_exporter_to_console((unsigned char*)argv[3], fs_repo);
int retVal = ipfs_exporter_to_console((unsigned char*)argv[3], local_node);
ipfs_node_free(local_node);
return retVal;
}
int ipfs_exporter_cat_node(struct Node* node, const struct FSRepo* fs_repo) {
int ipfs_exporter_cat_node(struct HashtableNode* node, struct IpfsNode* local_node) {
// process this node, then move on to the links
// build the unixfs
@ -183,12 +223,12 @@ int ipfs_exporter_cat_node(struct Node* node, const struct FSRepo* fs_repo) {
struct NodeLink* current = node->head_link;
while (current != NULL) {
// find the node
struct Node* child_node = NULL;
if (ipfs_merkledag_get(current->hash, current->hash_size, &child_node, fs_repo) == 0) {
struct HashtableNode* child_node = NULL;
if (!ipfs_exporter_get_node(local_node, current->hash, current->hash_size, &child_node)) {
return 0;
}
ipfs_exporter_cat_node(child_node, fs_repo);
ipfs_node_free(child_node);
ipfs_exporter_cat_node(child_node, local_node);
ipfs_hashtable_node_free(child_node);
current = current->next;
}
@ -202,7 +242,7 @@ int ipfs_exporter_cat_node(struct Node* node, const struct FSRepo* fs_repo) {
* @returns true(1) on success
*/
int ipfs_exporter_object_cat(int argc, char** argv) {
struct FSRepo* fs_repo = NULL;
struct IpfsNode *local_node = NULL;
char* repo_dir = NULL;
if (!ipfs_repo_get_directory(argc, argv, &repo_dir)) {
@ -210,15 +250,9 @@ int ipfs_exporter_object_cat(int argc, char** argv) {
return 0;
}
// open the repo
int retVal = ipfs_repo_fsrepo_new(repo_dir, NULL, &fs_repo);
if (retVal == 0) {
if (!ipfs_node_online_new(repo_dir, &local_node))
return 0;
}
retVal = ipfs_repo_fsrepo_open(fs_repo);
if (retVal == 0) {
return 0;
}
// find hash
// convert hash to cid
struct Cid* cid = NULL;
@ -227,16 +261,16 @@ int ipfs_exporter_object_cat(int argc, char** argv) {
}
// find block
struct Node* read_node = NULL;
if (ipfs_merkledag_get(cid->hash, cid->hash_length, &read_node, fs_repo) == 0) {
struct HashtableNode* read_node = NULL;
if (ipfs_exporter_get_node(local_node, cid->hash, cid->hash_length, &read_node)) {
ipfs_cid_free(cid);
return 0;
}
// no longer need the cid
ipfs_cid_free(cid);
ipfs_exporter_cat_node(read_node, fs_repo);
ipfs_node_free(read_node);
int retVal = ipfs_exporter_cat_node(read_node, local_node);
ipfs_hashtable_node_free(read_node);
return retVal;

View File

@ -5,6 +5,7 @@
#include "ipfs/importer/importer.h"
#include "ipfs/merkledag/merkledag.h"
#include "libp2p/os/utils.h"
#include "ipfs/core/ipfs_node.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
#include "ipfs/repo/init.h"
#include "ipfs/unixfs/unixfs.h"
@ -22,7 +23,7 @@
* @param blocksize the blocksize to add
* @returns true(1) on success
*/
int ipfs_importer_add_filesize_to_data_section(struct Node* node, size_t bytes_read) {
int ipfs_importer_add_filesize_to_data_section(struct HashtableNode* node, size_t bytes_read) {
// now add to the data section
struct UnixFS* data_section = NULL;
if (node->data == NULL) {
@ -41,7 +42,7 @@ int ipfs_importer_add_filesize_to_data_section(struct Node* node, size_t bytes_r
unsigned char protobuf[protobuf_size];
ipfs_unixfs_protobuf_encode(data_section, protobuf, protobuf_size, &protobuf_size);
ipfs_unixfs_free(data_section);
ipfs_node_set_data(node, protobuf, protobuf_size);
ipfs_hashtable_node_set_data(node, protobuf, protobuf_size);
return 1;
}
@ -51,13 +52,13 @@ int ipfs_importer_add_filesize_to_data_section(struct Node* node, size_t bytes_r
* @param node the node to add to
* @returns number of bytes read
*/
size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs_repo, size_t* total_size, size_t* bytes_written) {
size_t ipfs_import_chunk(FILE* file, struct HashtableNode* parent_node, struct FSRepo* fs_repo, size_t* total_size, size_t* bytes_written) {
unsigned char buffer[MAX_DATA_SIZE];
size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file);
// structs used by this method
struct UnixFS* new_unixfs = NULL;
struct Node* new_node = NULL;
struct HashtableNode* new_node = NULL;
struct NodeLink* new_link = NULL;
// put the file bits into a new UnixFS file
@ -90,63 +91,63 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
// if there is more to read, create a new node.
if (bytes_read == MAX_DATA_SIZE) {
// create a new node
if (ipfs_node_new_from_data(protobuf, *bytes_written, &new_node) == 0) {
if (ipfs_hashtable_node_new_from_data(protobuf, *bytes_written, &new_node) == 0) {
return 0;
}
// persist
size_t size_of_node = 0;
if (ipfs_merkledag_add(new_node, fs_repo, &size_of_node) == 0) {
ipfs_node_free(new_node);
ipfs_hashtable_node_free(new_node);
return 0;
}
// put link in parent node
if (ipfs_node_link_create(NULL, new_node->hash, new_node->hash_size, &new_link) == 0) {
ipfs_node_free(new_node);
ipfs_hashtable_node_free(new_node);
return 0;
}
new_link->t_size = size_of_node;
*total_size += new_link->t_size;
// NOTE: disposal of this link object happens when the parent is disposed
if (ipfs_node_add_link(parent_node, new_link) == 0) {
ipfs_node_free(new_node);
if (ipfs_hashtable_node_add_link(parent_node, new_link) == 0) {
ipfs_hashtable_node_free(new_node);
return 0;
}
ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read);
ipfs_node_free(new_node);
ipfs_hashtable_node_free(new_node);
*bytes_written = size_of_node;
size_of_node = 0;
} else {
// if there are no existing links, put what we pulled from the file into parent_node
// otherwise, add it as a link
if (parent_node->head_link == NULL) {
ipfs_node_set_data(parent_node, protobuf, *bytes_written);
ipfs_hashtable_node_set_data(parent_node, protobuf, *bytes_written);
} else {
// there are existing links. put the data in a new node, save it, then put the link in parent_node
// create a new node
if (ipfs_node_new_from_data(protobuf, *bytes_written, &new_node) == 0) {
if (ipfs_hashtable_node_new_from_data(protobuf, *bytes_written, &new_node) == 0) {
return 0;
}
// persist
if (ipfs_merkledag_add(new_node, fs_repo, &size_of_node) == 0) {
ipfs_node_free(new_node);
ipfs_hashtable_node_free(new_node);
return 0;
}
// put link in parent node
if (ipfs_node_link_create(NULL, new_node->hash, new_node->hash_size, &new_link) == 0) {
ipfs_node_free(new_node);
ipfs_hashtable_node_free(new_node);
return 0;
}
new_link->t_size = size_of_node;
*total_size += new_link->t_size;
// NOTE: disposal of this link object happens when the parent is disposed
if (ipfs_node_add_link(parent_node, new_link) == 0) {
ipfs_node_free(new_node);
if (ipfs_hashtable_node_add_link(parent_node, new_link) == 0) {
ipfs_hashtable_node_free(new_node);
return 0;
}
ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read);
ipfs_node_free(new_node);
ipfs_hashtable_node_free(new_node);
}
// persist the main node
ipfs_merkledag_add(parent_node, fs_repo, bytes_written);
@ -162,7 +163,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
* @param file_name the name of the file
* @returns true(1) if successful, false(0) if couldn't generate the MultiHash to be displayed
*/
int ipfs_import_print_node_results(const struct Node* node, const char* file_name) {
int ipfs_import_print_node_results(const struct HashtableNode* node, const char* file_name) {
// give some results to the user
//TODO: if directory_entry is itself a directory, traverse and report files
int buffer_len = 100;
@ -191,7 +192,7 @@ int ipfs_import_print_node_results(const struct Node* node, const char* file_nam
* @param recursive true if we should navigate directories
* @returns true(1) on success
*/
int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** parent_node, struct FSRepo* fs_repo, size_t* bytes_written, int recursive) {
int ipfs_import_file(const char* root_dir, const char* fileName, struct HashtableNode** parent_node, struct IpfsNode* local_node, size_t* bytes_written, int recursive) {
/**
* NOTE: When this function completes, parent_node will be either:
* 1) the complete file, in the case of a small file (<256k-ish)
@ -217,7 +218,7 @@ int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** p
new_root_dir = path;
}
// initialize parent_node as a directory
if (ipfs_node_create_directory(parent_node) == 0) {
if (ipfs_hashtable_node_create_directory(parent_node) == 0) {
if (path != NULL)
free(path);
if (file != NULL)
@ -231,7 +232,7 @@ int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** p
while (next != NULL) {
// process each file. NOTE: could be an embedded directory
*bytes_written = 0;
struct Node* file_node;
struct HashtableNode* file_node;
// put the filename together from fileName, which is the directory, and next->file_name
// which is a file (or a directory) within the directory we just found.
size_t filename_len = strlen(fileName) + strlen(next->file_name) + 2;
@ -239,8 +240,8 @@ int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** p
os_utils_filepath_join(fileName, next->file_name, full_file_name, filename_len);
// adjust root directory
if (ipfs_import_file(new_root_dir, full_file_name, &file_node, fs_repo, bytes_written, recursive) == 0) {
ipfs_node_free(*parent_node);
if (ipfs_import_file(new_root_dir, full_file_name, &file_node, local_node, bytes_written, recursive) == 0) {
ipfs_hashtable_node_free(*parent_node);
os_utils_free_file_list(first);
if (file != NULL)
free(file);
@ -259,16 +260,16 @@ int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** p
ipfs_node_link_create(next->file_name, file_node->hash, file_node->hash_size, &file_node_link);
file_node_link->t_size = *bytes_written;
// add file_node as link to parent_node
ipfs_node_add_link(*parent_node, file_node_link);
ipfs_hashtable_node_add_link(*parent_node, file_node_link);
// clean up file_node
ipfs_node_free(file_node);
ipfs_hashtable_node_free(file_node);
// move to next file in list
next = next->next;
} // while going through files
}
// save the parent_node (the directory)
size_t bytes_written;
ipfs_merkledag_add(*parent_node, fs_repo, &bytes_written);
ipfs_merkledag_add(*parent_node, local_node->repo, &bytes_written);
if (file != NULL)
free(file);
if (path != NULL)
@ -277,7 +278,7 @@ int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** p
} else {
// process this file
FILE* file = fopen(fileName, "rb");
retVal = ipfs_node_new(parent_node);
retVal = ipfs_hashtable_node_new(parent_node);
if (retVal == 0) {
return 0;
}
@ -285,12 +286,15 @@ int ipfs_import_file(const char* root_dir, const char* fileName, struct Node** p
// add all nodes (will be called multiple times for large files)
while ( bytes_read == MAX_DATA_SIZE) {
size_t written = 0;
bytes_read = ipfs_import_chunk(file, *parent_node, fs_repo, &total_size, &written);
bytes_read = ipfs_import_chunk(file, *parent_node, local_node->repo, &total_size, &written);
*bytes_written += written;
}
fclose(file);
}
// notify the network
local_node->routing->Provide(local_node->routing, (*parent_node)->hash, (*parent_node)->hash_size);
return 1;
}
@ -359,51 +363,45 @@ int ipfs_import_files(int argc, char** argv) {
* param 2: -r (optional)
* param 3: directoryname
*/
struct FSRepo* fs_repo = NULL;
struct IpfsNode* local_node = NULL;
char* repo_path = NULL;
int retVal = 0;
int recursive = ipfs_import_is_recursive(argc, argv);
// parse the command line
struct FileList* first = ipfs_import_get_filelist(argc, argv);
// open the repo
char* repo_path = NULL;
if (!ipfs_repo_get_directory(argc, argv, &repo_path)) {
// dir doesn't exist
fprintf(stderr, "Repo does not exist: %s\n", repo_path);
return 0;
}
if (!ipfs_repo_fsrepo_new(repo_path, NULL, &fs_repo)) {
fprintf(stderr, "Unable to build the repo struct: %s\n", repo_path);
return 0;
}
if (!ipfs_repo_fsrepo_open(fs_repo)) {
fprintf(stderr, "Unable to open repository: %s\n", repo_path);
return 0;
}
ipfs_node_online_new(repo_path, &local_node);
int retVal = 0;
// import the file(s)
struct FileList* current = first;
while (current != NULL) {
struct Node* directory_entry = NULL;
struct HashtableNode* directory_entry = NULL;
char* path = NULL;
char* filename = NULL;
os_utils_split_filename(current->file_name, &path, &filename);
size_t bytes_written = 0;
retVal = ipfs_import_file(NULL, current->file_name, &directory_entry, fs_repo, &bytes_written, recursive);
retVal = ipfs_import_file(NULL, current->file_name, &directory_entry, local_node, &bytes_written, recursive);
ipfs_import_print_node_results(directory_entry, filename);
// cleanup
ipfs_node_free(directory_entry);
ipfs_hashtable_node_free(directory_entry);
if (path != NULL)
free(path);
free(filename);
current = current->next;
}
if (fs_repo != NULL)
ipfs_repo_fsrepo_free(fs_repo);
if (local_node!= NULL)
ipfs_node_free(local_node);
// free file list
current = first;
while (current != NULL) {

View File

@ -106,7 +106,7 @@ int ipfs_resolver_is_remote(const char* path, const struct FSRepo* fs_repo) {
* @param fs_repo the local repo
* @returns the node, or NULL if not found
*/
struct Node* ipfs_resolver_remote_get(const char* path, struct Node* from, const struct IpfsNode* ipfs_node) {
struct HashtableNode* ipfs_resolver_remote_get(const char* path, struct HashtableNode* from, const struct IpfsNode* ipfs_node) {
// parse the path
const char* temp = ipfs_resolver_remove_path_prefix(path, ipfs_node->repo);
if (temp == NULL)
@ -158,8 +158,8 @@ struct Node* ipfs_resolver_remote_get(const char* path, struct Node* from, const
if (response_size == 1)
return NULL;
// turn the protobuf into a Node
struct Node* node;
ipfs_node_protobuf_decode(response, response_size, &node);
struct HashtableNode* node;
ipfs_hashtable_node_protobuf_decode(response, response_size, &node);
return node;
}
@ -170,7 +170,7 @@ struct Node* ipfs_resolver_remote_get(const char* path, struct Node* from, const
* @param from the current node (or NULL if it is the first call)
* @returns what we are looking for, or NULL if it wasn't found
*/
struct Node* ipfs_resolver_get(const char* path, struct Node* from, const struct IpfsNode* ipfs_node) {
struct HashtableNode* ipfs_resolver_get(const char* path, struct HashtableNode* from, const struct IpfsNode* ipfs_node) {
struct FSRepo* fs_repo = ipfs_node->repo;
@ -191,7 +191,7 @@ struct Node* ipfs_resolver_get(const char* path, struct Node* from, const struct
char* path_section;
if (ipfs_resolver_next_path(path, &path_section) == 0)
return NULL;
struct Node* current_node = NULL;
struct HashtableNode* current_node = NULL;
if (from == NULL) {
// this is the first time around. Grab the root node
if (path_section[0] == 'Q' && path_section[1] == 'm') {
@ -215,7 +215,7 @@ struct Node* ipfs_resolver_get(const char* path, struct Node* from, const struct
} else {
// look on...
free(path_section);
struct Node* newNode = ipfs_resolver_get(&path[pos+1], current_node, ipfs_node); // the +1 is the slash
struct HashtableNode* newNode = ipfs_resolver_get(&path[pos+1], current_node, ipfs_node); // the +1 is the slash
return newNode;
}
} else {
@ -225,7 +225,7 @@ struct Node* ipfs_resolver_get(const char* path, struct Node* from, const struct
}
} else {
// we were passed a node. If it is a directory, see if what we're looking for is in it
if (ipfs_node_is_directory(from)) {
if (ipfs_hashtable_node_is_directory(from)) {
struct NodeLink* curr_link = from->head_link;
while (curr_link != NULL) {
// if it matches the name, we found what we're looking for.
@ -237,7 +237,7 @@ struct Node* ipfs_resolver_get(const char* path, struct Node* from, const struct
}
if (strlen(path_section) == strlen(path)) {
// we are at the end of our search
ipfs_node_free(from);
ipfs_hashtable_node_free(from);
free(path_section);
return current_node;
} else {
@ -246,8 +246,8 @@ struct Node* ipfs_resolver_get(const char* path, struct Node* from, const struct
free(path_section);
// if we're at the end of the path, return the node
// continue looking for the next part of the path
ipfs_node_free(from);
struct Node* newNode = ipfs_resolver_get(next_path_section, current_node, ipfs_node);
ipfs_hashtable_node_free(from);
struct HashtableNode* newNode = ipfs_resolver_get(next_path_section, current_node, ipfs_node);
return newNode;
}
}
@ -262,7 +262,7 @@ struct Node* ipfs_resolver_get(const char* path, struct Node* from, const struct
// it should never get here
free(path_section);
if (from != NULL)
ipfs_node_free(from);
ipfs_hashtable_node_free(from);
return NULL;
}
@ -298,7 +298,7 @@ struct Libp2pPeer* ipfs_resolver_find_peer(const char* path, const struct IpfsNo
// ask the swarm for the peer
const char* address_string = ipfs_resolver_remove_path_prefix(path, fs_repo);
ipfs_node->routing->FindPeer(ipfs_node->routing, address_string, strlen(address_string), &peer);
ipfs_node->routing->FindPeer(ipfs_node->routing, (const unsigned char*)address_string, strlen(address_string), &peer);
return peer;
}

View File

@ -59,7 +59,7 @@ int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, st
/**
* Put a struct Node in the blockstore
*/
int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written);
int ipfs_blockstore_get_node(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo);
int ipfs_blockstore_put_node(const struct HashtableNode* node, const struct FSRepo* fs_repo, size_t* bytes_written);
int ipfs_blockstore_get_node(const unsigned char* hash, size_t hash_length, struct HashtableNode** node, const struct FSRepo* fs_repo);
#endif

View File

@ -5,6 +5,7 @@
#include "ipfs/commands/context.h"
#include "ipfs/repo/config/config.h"
#include "ipfs/core/ipfs_node.h"
struct BuildCfg {
int online;

View File

@ -19,3 +19,17 @@ struct IpfsNode {
//struct Mount** mounts;
// TODO: Add more here
};
/***
* build an online IpfsNode
* @param repo_path where the IPFS repository directory is
* @param node the completed IpfsNode struct
* @returns true(1) on success
*/
int ipfs_node_online_new(const char* repo_path, struct IpfsNode** node);
/***
* Free resources from the creation of an IpfsNode
* @param node the node to free
* @returns true(1)
*/
int ipfs_node_free(struct IpfsNode* node);

View File

@ -1,5 +1,11 @@
#pragma once
#include "ipfs/core/ipfs_node.h"
/**
* Pull bytes from the hashtable
*/
/**
* get a file by its hash, and write the data to a file
* @param hash the base58 multihash of the cid
@ -8,6 +14,16 @@
*/
int ipfs_exporter_to_file(const unsigned char* hash, const char* file_name, const struct FSRepo* fs_repo);
/***
* Retrieve a protobuf'd Node from the router
* @param local_node the context
* @param hash the hash to retrieve
* @param hash_size the length of the hash
* @param result a place to store the Node
* @returns true(1) on success, otherwise false(0)
*/
int ipfs_exporter_get_node(struct IpfsNode* local_node, const unsigned char* hash, const size_t hash_size, struct HashtableNode** result);
int ipfs_exporter_object_get(int argc, char** argv);
/***

View File

@ -2,7 +2,7 @@
#define __IPFS_IMPORTER_IMPORTER_H__
#include "ipfs/merkledag/node.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
#include "ipfs/core/ipfs_node.h"
/**
* Creates a node based on an incoming file or directory
@ -19,7 +19,7 @@
* @param recursive true if we should navigate directories
* @returns true(1) on success
*/
int ipfs_import_file(const char* root, const char* fileName, struct Node** parent_node, struct FSRepo* fs_repo, size_t* bytes_written, int recursive);
int ipfs_import_file(const char* root, const char* fileName, struct HashtableNode** parent_node, struct IpfsNode *local_node, size_t* bytes_written, int recursive);
/**
* called from the command line

View File

@ -14,7 +14,7 @@
* @param from the current node (or NULL if it is the first call)
* @returns what we are looking for, or NULL if it wasn't found
*/
struct Node* ipfs_resolver_get(const char* path, struct Node* from, const struct IpfsNode* ipfs_node);
struct HashtableNode* ipfs_resolver_get(const char* path, struct HashtableNode* from, const struct IpfsNode* ipfs_node);
/**
* Interrogate the path, looking for the peer

View File

@ -43,7 +43,7 @@ int main(int argc, char** argv)
////Nodes/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//N_Create_From_Link
struct Node * Mynode;
struct HashtableNode * Mynode;
Mynode = N_Create_From_Link(mylink);
mylink->name = "HAHA";//Testing for valid node creation
printf("Node Link[0] Name: %s\nHash: %s\n",Mynode->head_link[0]->name, Mynode->head_link[0]->Lcid->hash);
@ -72,7 +72,7 @@ int main(int argc, char** argv)
printf("Outlinkamt: %d\n", Mynode->link_ammount);
//Node Copy
struct Node * Node2;
struct HashtableNode * Node2;
Node2 = Node_Copy(Mynode);
printf("NODE COPY TEST: [0]: %s\n", Node2->head_link[0]->Lcid->hash);
Node_Delete(Node2);

View File

@ -14,7 +14,7 @@
* @param bytes_written the number of bytes written
* @returns true(1) on success
*/
int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_written);
int ipfs_merkledag_add(struct HashtableNode* node, struct FSRepo* fs_repo, size_t* bytes_written);
/***
* Retrieves a node from the datastore based on the cid
@ -23,7 +23,7 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_
* @param fs_repo the repository
* @returns true(1) on success
*/
int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node** node, const struct FSRepo* fs_repo);
int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct HashtableNode** node, const struct FSRepo* fs_repo);
/***
* Retrieves a node from the datastore based on the multihash
@ -32,6 +32,6 @@ int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node*
* @param fs_repo the repository
* @returns true(1) on success
*/
int ipfs_merkledag_get_by_multihash(const unsigned char* multihash, size_t multihash_length, struct Node** node, const struct FSRepo* fs_repo);
int ipfs_merkledag_get_by_multihash(const unsigned char* multihash, size_t multihash_length, struct HashtableNode** node, const struct FSRepo* fs_repo);
#endif

View File

@ -22,7 +22,7 @@ struct NodeLink
struct NodeLink* next;
};
struct Node
struct HashtableNode
{
// saved in protobuf
size_t data_size;
@ -103,7 +103,7 @@ int ipfs_node_link_protobuf_decode(unsigned char* buffer, size_t buffer_length,
* @param node the node to examine
* @returns the max size of an encoded stream of bytes, if it were encoded
*/
size_t ipfs_node_protobuf_encode_size(const struct Node* node);
size_t ipfs_hashtable_node_protobuf_encode_size(const struct HashtableNode* node);
/***
* Encode a node into a protobuf byte stream
@ -113,7 +113,7 @@ size_t ipfs_node_protobuf_encode_size(const struct Node* node);
* @param bytes_written how much of buffer was used
* @returns true(1) on success
*/
int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
int ipfs_hashtable_node_protobuf_encode(const struct HashtableNode* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
/***
* Decode a stream of bytes into a Node structure
@ -122,7 +122,7 @@ int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, si
* @param node pointer to the Node to be created
* @returns true(1) on success
*/
int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struct Node** node);
int ipfs_hashtable_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struct HashtableNode** node);
/*====================================================================================
* Node Functions
@ -133,7 +133,7 @@ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struc
* @param node the pointer to the memory allocated
* @returns true(1) on success, otherwise false(0)
*/
int ipfs_node_new(struct Node** node);
int ipfs_hashtable_node_new(struct HashtableNode** node);
/***
* Allocates memory for a node, and sets the data section to indicate
@ -141,14 +141,14 @@ int ipfs_node_new(struct Node** node);
* @param node the node to initialize
* @returns true(1) on success, otherwise false(0)
*/
int ipfs_node_create_directory(struct Node** node);
int ipfs_hashtable_node_create_directory(struct HashtableNode** node);
/***
* Determine if this node is actually a directory
* @param node the node to examine
* @returns true(1) if this node is a directory. Otherwise, false(0)
*/
int ipfs_node_is_directory(struct Node* node);
int ipfs_hashtable_node_is_directory(struct HashtableNode* node);
/**
* sets the Cid into the struct element titled cached
@ -156,7 +156,7 @@ int ipfs_node_is_directory(struct Node* node);
* @param cid the cid
* @returns true(1) on success
*/
int ipfs_node_set_hash(struct Node* node, const unsigned char* hash, size_t hash_size);
int ipfs_hashtable_node_set_hash(struct HashtableNode* node, const unsigned char* hash, size_t hash_size);
/*ipfs_node_set_data
* Sets the data of a node
@ -165,42 +165,42 @@ int ipfs_node_set_hash(struct Node* node, const unsigned char* hash, size_t hash
* Sets pointers of encoded & cached to NULL /following go method
* returns 1 on success 0 on failure
*/
int ipfs_node_set_data(struct Node * N, unsigned char * Data, size_t data_size);
int ipfs_hashtable_node_set_data(struct HashtableNode * N, unsigned char * Data, size_t data_size);
/*ipfs_node_set_encoded
* @param NODE: the node you wish to alter (struct Node *)
* @param Data: The data you wish to set in encoded.(unsigned char *)
* returns 1 on success 0 on failure
*/
int ipfs_node_set_encoded(struct Node * N, unsigned char * Data);
int ipfs_hashtable_node_set_encoded(struct HashtableNode * N, unsigned char * Data);
/*ipfs_node_get_data
* Gets data from a node
* @param Node: = The node you want to get data from. (unsigned char *)
* Returns data of node.
*/
unsigned char * ipfs_node_get_data(struct Node * N);
unsigned char * ipfs_hashtable_node_get_data(struct HashtableNode * N);
/*ipfs_node_free
* Once you are finished using a node, always delete it using this.
* It will take care of the links inside it.
* @param N: the node you want to free. (struct Node *)
*/
int ipfs_node_free(struct Node * N);
int ipfs_hashtable_node_free(struct HashtableNode * N);
/*ipfs_node_get_link_by_name
* Returns a copy of the link with given name
* @param Name: (char * name) searches for link with this name
* Returns the link struct if it's found otherwise returns NULL
*/
struct NodeLink * ipfs_node_get_link_by_name(struct Node * N, char * Name);
struct NodeLink * ipfs_hashtable_node_get_link_by_name(struct HashtableNode * N, char * Name);
/*ipfs_node_remove_link_by_name
* Removes a link from node if found by name.
* @param name: Name of link (char * name)
* returns 1 on success, 0 on failure.
*/
int ipfs_node_remove_link_by_name(char * Name, struct Node * mynode);
int ipfs_hashtable_node_remove_link_by_name(char * Name, struct HashtableNode * mynode);
/* ipfs_node_add_link
* Adds a link to your node
@ -209,7 +209,7 @@ int ipfs_node_remove_link_by_name(char * Name, struct Node * mynode);
* @param linksz: sizeof(your cid here)
* Returns your node with the newly added link
*/
int ipfs_node_add_link(struct Node * mynode, struct NodeLink * mylink);
int ipfs_hashtable_node_add_link(struct HashtableNode * mynode, struct NodeLink * mylink);
/*ipfs_node_new_from_link
* Create a node from a link
@ -217,7 +217,7 @@ int ipfs_node_add_link(struct Node * mynode, struct NodeLink * mylink);
* @param node the pointer to the new node
* @returns true(1) on success
*/
int ipfs_node_new_from_link(struct NodeLink * mylink, struct Node** node);
int ipfs_hashtable_node_new_from_link(struct NodeLink * mylink, struct HashtableNode** node);
/*ipfs_node_new_from_data
* @param data: bytes buffer you want to create the node from
@ -225,7 +225,7 @@ int ipfs_node_new_from_link(struct NodeLink * mylink, struct Node** node);
* @param node the pointer to the new node
* @returns true(1) on success
*/
int ipfs_node_new_from_data(unsigned char * data, size_t data_size, struct Node** node);
int ipfs_hashtable_node_new_from_data(unsigned char * data, size_t data_size, struct HashtableNode** node);
/***
* create a Node struct from encoded data
@ -233,7 +233,7 @@ int ipfs_node_new_from_data(unsigned char * data, size_t data_size, struct Node*
* @param node a pointer to the node that will be created
* @returns true(1) on success
*/
int ipfs_node_new_from_encoded(unsigned char * data, struct Node** node);
int ipfs_hashtable_node_new_from_encoded(unsigned char * data, struct HashtableNode** node);
/*Node_Resolve_Max_Size
* !!!This shouldn't concern you!
@ -278,7 +278,7 @@ struct Link_Proc
* @param N: The node you want to get links from
* @param path: The "foo/bar/bin" path
*/
struct Link_Proc * Node_Resolve_Links(struct Node * N, char * path);
struct Link_Proc * Node_Resolve_Links(struct HashtableNode * N, char * path);
/*Free_link_Proc
* frees the Link_Proc struct you created.

View File

@ -33,6 +33,7 @@ struct Reprovider {
struct RepoConfig {
struct Identity* identity;
struct Datastore* datastore;
struct Filestore* filestore;
struct Addresses* addresses;
struct Mounts mounts;
struct Discovery discovery;

View File

@ -91,8 +91,8 @@ int ipfs_repo_fsrepo_unixfs_read(const unsigned char* hash, size_t hash_length,
* @param fs_repo the repo to write to
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_node_write(const struct Node* unix_fs, const struct FSRepo