Intermidiate commit with big changes to storage formats
I am attempting to match the storage format of the reference implementation, so as to generate the same hashes.yamux
jmjatlanta
parent
a569159cc2
commit
914d3caaed
2
Makefile
2
Makefile
|
|
@ -16,6 +16,7 @@ all:
|
|||
cd flatfs; make all;
|
||||
cd datastore; make all;
|
||||
cd thirdparty; make all;
|
||||
cd unixfs; make all;
|
||||
cd main; make all;
|
||||
cd test; make all;
|
||||
|
||||
|
|
@ -33,6 +34,7 @@ clean:
|
|||
cd flatfs; make clean;
|
||||
cd datastore; make clean;
|
||||
cd thirdparty; make clean;
|
||||
cd unixfs; make clean;
|
||||
cd main; make clean;
|
||||
cd test; make clean;
|
||||
|
||||
|
|
|
|||
|
|
@ -40,12 +40,12 @@ unsigned char* ipfs_blockstore_cid_to_base32(const struct Cid* cid) {
|
|||
return buffer;
|
||||
}
|
||||
|
||||
unsigned char* ipfs_blockstore_hash_to_base32(const unsigned char* hash, size_t hash_size) {
|
||||
size_t key_length = libp2p_crypto_encoding_base32_encode_size(hash_size);
|
||||
unsigned char* ipfs_blockstore_hash_to_base32(const unsigned char* hash, size_t hash_length) {
|
||||
size_t key_length = libp2p_crypto_encoding_base32_encode_size(hash_length);
|
||||
unsigned char* buffer = (unsigned char*)malloc(key_length + 1);
|
||||
if (buffer == NULL)
|
||||
return NULL;
|
||||
int retVal = ipfs_datastore_helper_ds_key_from_binary(hash, hash_size, &buffer[0], key_length, &key_length);
|
||||
int retVal = ipfs_datastore_helper_ds_key_from_binary(hash, hash_length, &buffer[0], key_length, &key_length);
|
||||
if (retVal == 0) {
|
||||
free(buffer);
|
||||
return NULL;
|
||||
|
|
@ -145,3 +145,135 @@ int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) {
|
|||
free(filename);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/***
|
||||
* Put a struct UnixFS in the blockstore
|
||||
* @param unix_fs the structure
|
||||
* @param fs_repo the repo to place the strucure in
|
||||
* @param bytes_written the number of bytes written to the blockstore
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blockstore_put_unixfs(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written) {
|
||||
// from blockstore.go line 118
|
||||
int retVal = 0;
|
||||
|
||||
// Get Datastore key, which is a base32 key of the multihash,
|
||||
unsigned char* key = ipfs_blockstore_hash_to_base32(unix_fs->hash, unix_fs->hash_length);
|
||||
if (key == NULL) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//TODO: put this in subdirectories
|
||||
|
||||
// turn the block into a binary array
|
||||
size_t protobuf_len = ipfs_unixfs_protobuf_encode_size(unix_fs);
|
||||
unsigned char protobuf[protobuf_len];
|
||||
retVal = ipfs_unixfs_protobuf_encode(unix_fs, protobuf, protobuf_len, &protobuf_len);
|
||||
if (retVal == 0) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// now write byte array to file
|
||||
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
|
||||
if (filename == NULL) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
FILE* file = fopen(filename, "wb");
|
||||
*bytes_written = fwrite(protobuf, 1, protobuf_len, file);
|
||||
fclose(file);
|
||||
if (*bytes_written != protobuf_len) {
|
||||
free(key);
|
||||
free(filename);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// send to Put with key (this is now done separately)
|
||||
//fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
|
||||
|
||||
free(key);
|
||||
free(filename);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/***
|
||||
* Find a UnixFS struct based on its hash
|
||||
* @param hash the hash to look for
|
||||
* @param hash_length the length of the hash
|
||||
* @param unix_fs the struct to fill
|
||||
* @param fs_repo where to look for the data
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, struct UnixFS** block, const struct FSRepo* fs_repo) {
|
||||
// get datastore key, which is a base32 key of the multihash
|
||||
unsigned char* key = ipfs_blockstore_hash_to_base32(hash, hash_length);
|
||||
|
||||
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
|
||||
|
||||
size_t file_size = os_utils_file_size(filename);
|
||||
unsigned char buffer[file_size];
|
||||
|
||||
FILE* file = fopen(filename, "rb");
|
||||
size_t bytes_read = fread(buffer, 1, file_size, file);
|
||||
fclose(file);
|
||||
|
||||
int retVal = ipfs_unixfs_protobuf_decode(buffer, bytes_read, block);
|
||||
|
||||
free(key);
|
||||
free(filename);
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
/***
|
||||
* Put a struct Node in the blockstore
|
||||
* @param node the structure
|
||||
* @param fs_repo the repo to place the strucure in
|
||||
* @param bytes_written the number of bytes written to the blockstore
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written) {
|
||||
// from blockstore.go line 118
|
||||
int retVal = 0;
|
||||
|
||||
// Get Datastore key, which is a base32 key of the multihash,
|
||||
unsigned char* key = ipfs_blockstore_hash_to_base32(node->hash, node->hash_size);
|
||||
if (key == NULL) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//TODO: put this in subdirectories
|
||||
|
||||
// turn the block into a binary array
|
||||
size_t protobuf_len = ipfs_node_protobuf_encode_size(node);
|
||||
unsigned char protobuf[protobuf_len];
|
||||
retVal = ipfs_node_protobuf_encode(node, protobuf, protobuf_len, &protobuf_len);
|
||||
if (retVal == 0) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// now write byte array to file
|
||||
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
|
||||
if (filename == NULL) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
FILE* file = fopen(filename, "wb");
|
||||
*bytes_written = fwrite(protobuf, 1, protobuf_len, file);
|
||||
fclose(file);
|
||||
if (*bytes_written != protobuf_len) {
|
||||
free(key);
|
||||
free(filename);
|
||||
return 0;
|
||||
}
|
||||
|
||||
free(key);
|
||||
free(filename);
|
||||
return 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,27 +23,43 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
|
|||
unsigned char buffer[MAX_DATA_SIZE];
|
||||
size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file);
|
||||
|
||||
// put the file bits into a new UnixFS file
|
||||
struct UnixFS* new_unixfs = NULL;
|
||||
ipfs_unixfs_new(&new_unixfs);
|
||||
new_unixfs->data_type = UNIXFS_FILE;
|
||||
ipfs_unixfs_add_data(&buffer[0], bytes_read, new_unixfs);
|
||||
// protobuf the UnixFS
|
||||
size_t protobuf_size = ipfs_unixfs_protobuf_encode_size(new_unixfs);
|
||||
unsigned char protobuf[protobuf_size];
|
||||
size_t bytes_written = 0;
|
||||
ipfs_unixfs_protobuf_encode(new_unixfs, protobuf, protobuf_size, &bytes_written);
|
||||
// we're done with the object
|
||||
ipfs_unixfs_free(new_unixfs);
|
||||
// create a new node
|
||||
struct Node* new_node = NULL;
|
||||
ipfs_node_new_from_data(buffer, bytes_read, &new_node);
|
||||
ipfs_node_new_from_data(protobuf, bytes_written, &new_node);
|
||||
ipfs_node_set_hash(new_node, new_unixfs->hash, new_unixfs->hash_length);
|
||||
// persist
|
||||
ipfs_merkledag_add(new_node, fs_repo);
|
||||
size_t size_of_node = 0;
|
||||
ipfs_merkledag_add(new_node, fs_repo, &size_of_node);
|
||||
// put link in parent node
|
||||
struct NodeLink* new_link = NULL;
|
||||
ipfs_node_link_create("", new_node->hash, new_node->hash_size, &new_link);
|
||||
new_link->t_size = new_node->data_size;
|
||||
new_link->t_size = size_of_node;
|
||||
*total_size += new_link->t_size;
|
||||
ipfs_node_add_link(parent_node, new_link);
|
||||
ipfs_node_free(new_node);
|
||||
// save the parent_node if it is time...
|
||||
if (bytes_read != MAX_DATA_SIZE) {
|
||||
// build UnixFS for file
|
||||
// We have read everything, now save the parent_node,
|
||||
// which is a sort of "directory" entry
|
||||
/*
|
||||
// build UnixFS for the parent
|
||||
struct UnixFS* unix_fs;
|
||||
if (ipfs_unixfs_new(&unix_fs) == 0) {
|
||||
return 0;
|
||||
}
|
||||
unix_fs->data_type = UNIXFS_FILE;
|
||||
unix_fs->file_size = *total_size;
|
||||
unix_fs->bytes_size = *total_size;
|
||||
// now encode unixfs and put in parent_node->data
|
||||
size_t temp_size = ipfs_unixfs_protobuf_encode_size(unix_fs);
|
||||
unsigned char temp[temp_size];
|
||||
|
|
@ -59,8 +75,10 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
|
|||
return 0;
|
||||
}
|
||||
memcpy(parent_node->data, temp, bytes_written);
|
||||
ipfs_unixfs_free(unix_fs);
|
||||
*/
|
||||
// persist the main node
|
||||
ipfs_merkledag_add(parent_node, fs_repo);
|
||||
ipfs_merkledag_add(parent_node, fs_repo, &bytes_written);
|
||||
}
|
||||
return bytes_read;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,5 +37,28 @@ int ipfs_blockstore_get(const unsigned char* hash, size_t hash_length, struct Bl
|
|||
*/
|
||||
int ipfs_blockstore_put(struct Block* block, const struct FSRepo* fs_repo);
|
||||
|
||||
/***
|
||||
* Put a struct UnixFS in the blockstore
|
||||
* @param unix_fs the structure
|
||||
* @param fs_repo the repo to place the strucure in
|
||||
* @param bytes_written the number of bytes written to the blockstore
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blockstore_put_unixfs(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written);
|
||||
|
||||
/***
|
||||
* Find a UnixFS struct based on its hash
|
||||
* @param hash the hash to look for
|
||||
* @param hash_length the length of the hash
|
||||
* @param unix_fs the struct to fill
|
||||
* @param fs_repo where to look for the data
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, struct UnixFS** block, const struct FSRepo* fs_repo);
|
||||
|
||||
/**
|
||||
* Put a struct Node in the blockstore
|
||||
*/
|
||||
int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -10,10 +10,11 @@
|
|||
/***
|
||||
* Adds a node to the dagService and blockService
|
||||
* @param node the node to add
|
||||
* @param cid the resultant cid that was added
|
||||
* @param fs_repo the repo to add to
|
||||
* @param bytes_written the number of bytes written
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo);
|
||||
int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_written);
|
||||
|
||||
/***
|
||||
* Retrieves a node from the datastore based on the cid
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ int ipfs_node_link_free(struct NodeLink * node_link);
|
|||
* @param link the link to examine
|
||||
* @returns the maximum size that should be needed
|
||||
*/
|
||||
size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link);
|
||||
size_t ipfs_node_link_protobuf_encode_size(const struct NodeLink* link);
|
||||
|
||||
/***
|
||||
* Encode a NodeLink into protobuf format
|
||||
|
|
@ -84,7 +84,7 @@ size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link);
|
|||
* @pram bytes_written the amount of the buffer used
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_node_link_protobuf_encode(struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
|
||||
int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
|
||||
|
||||
/****
|
||||
* Decode from a byte array into a NodeLink
|
||||
|
|
@ -102,7 +102,7 @@ int ipfs_node_link_protobuf_decode(unsigned char* buffer, size_t buffer_length,
|
|||
* @param node the node to examine
|
||||
* @returns the max size of an encoded stream of bytes, if it were encoded
|
||||
*/
|
||||
size_t ipfs_node_protobuf_encode_size(struct Node* node);
|
||||
size_t ipfs_node_protobuf_encode_size(const struct Node* node);
|
||||
|
||||
/***
|
||||
* Encode a node into a protobuf byte stream
|
||||
|
|
@ -112,7 +112,7 @@ size_t ipfs_node_protobuf_encode_size(struct Node* node);
|
|||
* @param bytes_written how much of buffer was used
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
|
||||
int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
|
||||
|
||||
/***
|
||||
* Decode a stream of bytes into a Node structure
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@
|
|||
#define fs_repo_h
|
||||
|
||||
#include <stdio.h>
|
||||
#include "ipfs/repo/config/config.h"
|
||||
|
||||
#include "ipfs/repo/config/config.h"
|
||||
#include "ipfs/unixfs/unixfs.h"
|
||||
#include "ipfs/merkledag/node.h"
|
||||
|
||||
/**
|
||||
* a structure to hold the repo info
|
||||
|
|
@ -73,4 +75,23 @@ int ipfs_repo_fsrepo_init(struct FSRepo* config);
|
|||
int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo);
|
||||
int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo);
|
||||
|
||||
/***
|
||||
* Write a unixfs to the datastore and blockstore
|
||||
* @param unix_fs the struct to write
|
||||
* @param fs_repo the repo to write to
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_repo_fsrepo_unixfs_write(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written);
|
||||
int ipfs_repo_fsrepo_unixfs_read(const unsigned char* hash, size_t hash_length, struct UnixFS** unix_fs, const struct FSRepo* fs_repo);
|
||||
|
||||
/***
|
||||
* Write a struct Node to the datastore and blockstore
|
||||
* @param node the struct to write
|
||||
* @param fs_repo the repo to write to
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_repo_fsrepo_node_write(const struct Node* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written);
|
||||
int ipfs_repo_fsrepo_node_read(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo);
|
||||
|
||||
|
||||
#endif /* fs_repo_h */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
enum UnixFSFormatType { RAW, FILE, DIRECTORY, METADATA, SYMLINK };
|
||||
enum UnixFSFormatType { RAW, UNIXFS_FILE, DIRECTORY, METADATA, SYMLINK };
|
||||
|
||||
struct UnixFSData {
|
||||
enum UnixFSFormatType type;
|
||||
|
|
|
|||
|
|
@ -43,8 +43,10 @@ struct UnixFS {
|
|||
enum UnixFSDataType data_type;
|
||||
size_t bytes_size; // the size of the bytes array
|
||||
unsigned char* bytes; // an array of bytes
|
||||
size_t file_size; // the file size
|
||||
// size_t file_size; // the file size - I mimick this one
|
||||
struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes
|
||||
unsigned char* hash; // not saved
|
||||
size_t hash_length; // not saved
|
||||
};
|
||||
|
||||
struct UnixFSMetaData {
|
||||
|
|
@ -65,6 +67,15 @@ int ipfs_unixfs_new(struct UnixFS** obj);
|
|||
*/
|
||||
int ipfs_unixfs_free(struct UnixFS* obj);
|
||||
|
||||
/***
|
||||
* Write data to data section of a UnixFS stuct. NOTE: this also calculates a sha256 hash
|
||||
* @param data the data to write
|
||||
* @param data_length the length of the data
|
||||
* @param unix_fs the struct to add to
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs);
|
||||
|
||||
/**
|
||||
* Protobuf functions
|
||||
*/
|
||||
|
|
@ -74,7 +85,7 @@ int ipfs_unixfs_free(struct UnixFS* obj);
|
|||
* @param obj what will be encoded
|
||||
* @returns the size of the buffer necessary to encode the object
|
||||
*/
|
||||
size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj);
|
||||
size_t ipfs_unixfs_protobuf_encode_size(const struct UnixFS* obj);
|
||||
|
||||
/***
|
||||
* Encode a UnixFS object into protobuf format
|
||||
|
|
|
|||
|
|
@ -9,14 +9,14 @@ OBJS = main.o \
|
|||
../commands/argument.o ../commands/command_option.o ../commands/command.o ../commands/cli/parse.o \
|
||||
../core/builder.o \
|
||||
../datastore/ds_helper.o \
|
||||
../dnslink/dnslink.o \
|
||||
../flatfs/flatfs.o \
|
||||
../importer/importer.o \
|
||||
../importer/exporter.o \
|
||||
../dnslink/dnslink.o \
|
||||
../path/path.o \
|
||||
../namesys/isdomain.o \
|
||||
../merkledag/merkledag.o ../merkledag/node.o \
|
||||
../multibase/multibase.o \
|
||||
../namesys/isdomain.o \
|
||||
../os/utils.o \
|
||||
../repo/init.o \
|
||||
../repo/fsrepo/fs_repo.o ../repo/fsrepo/jsmn.o ../repo/fsrepo/lmdb_datastore.o \
|
||||
|
|
|
|||
|
|
@ -1,41 +1,46 @@
|
|||
/**
|
||||
* A basic storage building block of the IPFS system
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "libp2p/crypto/sha256.h"
|
||||
#include "ipfs/merkledag/merkledag.h"
|
||||
#include "ipfs/unixfs/unixfs.h"
|
||||
|
||||
/***
|
||||
* Adds a node to the dagService and blockService
|
||||
* @param node the node to add
|
||||
* @param cid the resultant cid that was added
|
||||
* @param fs_repo the repo to add to
|
||||
* @param bytes_written the number of bytes written
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
|
||||
int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_written) {
|
||||
// taken from merkledag.go line 59
|
||||
int retVal = 0;
|
||||
|
||||
struct Block* block = NULL;
|
||||
// compute the hash
|
||||
size_t protobuf_size = ipfs_node_protobuf_encode_size(node);
|
||||
unsigned char protobuf[protobuf_size];
|
||||
size_t bytes_encoded;
|
||||
retVal = ipfs_node_protobuf_encode(node, protobuf, protobuf_size, &bytes_encoded);
|
||||
|
||||
// protobuf the node
|
||||
size_t protobuf_len = ipfs_node_protobuf_encode_size(node);
|
||||
size_t bytes_written = 0;
|
||||
unsigned char protobuf[protobuf_len];
|
||||
ipfs_node_protobuf_encode(node, protobuf, protobuf_len, &bytes_written);
|
||||
|
||||
// turn the node into a block
|
||||
ipfs_blocks_block_new(&block);
|
||||
ipfs_blocks_block_add_data(protobuf, bytes_written, block);
|
||||
|
||||
// write to block store & datastore
|
||||
int retVal = ipfs_repo_fsrepo_block_write(block, fs_repo);
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
node->hash_size = 32;
|
||||
node->hash = (unsigned char*)malloc(node->hash_size);
|
||||
if (node->hash == NULL) {
|
||||
return 0;
|
||||
}
|
||||
if (libp2p_crypto_hashing_sha256(protobuf, bytes_encoded, &node->hash[0]) == 0) {
|
||||
free(node->hash);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ipfs_node_set_hash(node, block->cid->hash, block->cid->hash_length);
|
||||
|
||||
if (block != NULL)
|
||||
ipfs_blocks_block_free(block);
|
||||
// write to block store & datastore
|
||||
retVal = ipfs_repo_fsrepo_node_write(node, fs_repo, bytes_written);
|
||||
if (retVal == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// TODO: call HasBlock (unsure why as yet)
|
||||
return 1;
|
||||
|
|
@ -51,7 +56,8 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
|
|||
*/
|
||||
int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node** node, const struct FSRepo* fs_repo) {
|
||||
int retVal = 1;
|
||||
struct Block* block;
|
||||
//struct Block* block;
|
||||
struct UnixFS* unix_fs;
|
||||
size_t key_length = 100;
|
||||
unsigned char key[key_length];
|
||||
|
||||
|
|
@ -61,27 +67,27 @@ int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node*
|
|||
if (retVal == 0)
|
||||
return 0;
|
||||
|
||||
// we have the record from the db. Go get the block from the blockstore
|
||||
retVal = ipfs_repo_fsrepo_block_read(hash, hash_size, &block, fs_repo);
|
||||
// we have the record from the db. Go get the UnixFS from the blockstore
|
||||
retVal = ipfs_repo_fsrepo_unixfs_read(hash, hash_size, &unix_fs, fs_repo);
|
||||
if (retVal == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// now convert the block into a node
|
||||
if (ipfs_node_protobuf_decode(block->data, block->data_length, node) == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
if (ipfs_node_protobuf_decode(unix_fs->bytes, unix_fs->bytes_size, node) == 0) {
|
||||
ipfs_unixfs_free(unix_fs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// set the cid on the node
|
||||
if (ipfs_node_set_hash(*node, hash, hash_size) == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
ipfs_unixfs_free(unix_fs);
|
||||
ipfs_node_free(*node);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// free resources
|
||||
ipfs_blocks_block_free(block);
|
||||
ipfs_unixfs_free(unix_fs);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
|
||||
#include "ipfs/merkledag/node.h"
|
||||
|
||||
// for protobuf Node (all fields optional) links (repeated node_link) data (optional bytes)
|
||||
// for protobuf Node (all fields optional) data (optional bytes) links (repeated node_link)
|
||||
enum WireType ipfs_node_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED };
|
||||
// for protobuf NodeLink (all fields optional) hash name tsize
|
||||
enum WireType ipfs_node_link_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED, WIRETYPE_VARINT };
|
||||
|
|
@ -84,7 +84,7 @@ int ipfs_node_link_free(struct NodeLink * node_link)
|
|||
* @param link the link to examine
|
||||
* @returns the maximum length of the encoded NodeLink
|
||||
*/
|
||||
size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link) {
|
||||
size_t ipfs_node_link_protobuf_encode_size(const struct NodeLink* link) {
|
||||
if (link == NULL)
|
||||
return 0;
|
||||
// hash, name, tsize
|
||||
|
|
@ -109,7 +109,7 @@ size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link) {
|
|||
* @param bytes_written the number of bytes written to buffer
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_node_link_protobuf_encode(struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
|
||||
int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
|
||||
// 3 fields, hash (length delimited), name (length delimited), tsize (varint)
|
||||
size_t bytes_used = 0;
|
||||
int retVal = 0;
|
||||
|
|
@ -200,7 +200,7 @@ exit:
|
|||
/***
|
||||
* return an approximate size of the encoded node
|
||||
*/
|
||||
size_t ipfs_node_protobuf_encode_size(struct Node* node) {
|
||||
size_t ipfs_node_protobuf_encode_size(const struct Node* node) {
|
||||
size_t size = 0;
|
||||
// links
|
||||
struct NodeLink* current = node->head_link;
|
||||
|
|
@ -223,10 +223,17 @@ size_t ipfs_node_protobuf_encode_size(struct Node* node) {
|
|||
* @param bytes_written how much of buffer was used
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
|
||||
int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
|
||||
size_t bytes_used = 0;
|
||||
*bytes_written = 0;
|
||||
int retVal = 0;
|
||||
// data
|
||||
if (node->data_size > 0) {
|
||||
retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
*bytes_written += bytes_used;
|
||||
}
|
||||
// links
|
||||
struct NodeLink* current = node->head_link;
|
||||
while (current != NULL) {
|
||||
|
|
@ -235,19 +242,12 @@ int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t m
|
|||
retVal = ipfs_node_link_protobuf_encode(current, temp, temp_size, &bytes_used);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)temp, bytes_used, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
|
||||
retVal = protobuf_encode_length_delimited(2, ipfs_node_message_fields[1], (char*)temp, bytes_used, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
*bytes_written += bytes_used;
|
||||
current = current->next;
|
||||
}
|
||||
// data
|
||||
if (node->data_size > 0) {
|
||||
retVal = protobuf_encode_length_delimited(2, ipfs_node_message_fields[1], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
*bytes_written += bytes_used;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -261,8 +261,8 @@ int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t m
|
|||
*/
|
||||
int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struct Node** node) {
|
||||
/*
|
||||
* Field 0: link
|
||||
* Field 1: data
|
||||
* Field 2: link
|
||||
*/
|
||||
size_t pos = 0;
|
||||
int retVal = 0;
|
||||
|
|
@ -282,7 +282,13 @@ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struc
|
|||
}
|
||||
pos += bytes_read;
|
||||
switch(field_no) {
|
||||
case (1): {// links
|
||||
case (1): { // data
|
||||
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*node)->data), &((*node)->data_size), &bytes_read) == 0)
|
||||
goto exit;
|
||||
pos += bytes_read;
|
||||
break;
|
||||
}
|
||||
case (2): {// links
|
||||
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0)
|
||||
goto exit;
|
||||
pos += bytes_read;
|
||||
|
|
@ -293,12 +299,6 @@ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struc
|
|||
ipfs_node_add_link(*node, temp_link);
|
||||
break;
|
||||
}
|
||||
case (2): { // data
|
||||
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*node)->data), &((*node)->data_size), &bytes_read) == 0)
|
||||
goto exit;
|
||||
pos += bytes_read;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -554,6 +554,69 @@ int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_re
|
|||
return 1;
|
||||
}
|
||||
|
||||
/***
|
||||
* Write a unixfs to the datastore and blockstore
|
||||
* @param unix_fs the struct to write
|
||||
* @param fs_repo the repo to write to
|
||||
* @param bytes_written number of bytes written to the repo
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_repo_fsrepo_unixfs_write(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written) {
|
||||
/**
|
||||
* What is put in the blockstore is the block.
|
||||
* What is put in the datastore is the multihash (the Cid) as the key,
|
||||
* and the base32 encoded multihash as the value.
|
||||
*/
|
||||
int retVal = 1;
|
||||
retVal = ipfs_blockstore_put_unixfs(unix_fs, fs_repo, bytes_written);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
// take the hash, base32 it, and send both to the datastore
|
||||
size_t fs_key_length = 100;
|
||||
unsigned char fs_key[fs_key_length];
|
||||
retVal = ipfs_datastore_helper_ds_key_from_binary(unix_fs->hash, unix_fs->hash_length, fs_key, fs_key_length, &fs_key_length);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
retVal = fs_repo->config->datastore->datastore_put(unix_fs->hash, unix_fs->hash_length, fs_key, fs_key_length, fs_repo->config->datastore);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/***
|
||||
* Write a unixfs to the datastore and blockstore
|
||||
* @param unix_fs the struct to write
|
||||
* @param fs_repo the repo to write to
|
||||
* @param bytes_written number of bytes written to the repo
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_repo_fsrepo_node_write(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written) {
|
||||
/**
|
||||
* What is put in the blockstore is the node.
|
||||
* What is put in the datastore is the multihash as the key,
|
||||
* and the base32 encoded multihash as the value.
|
||||
*/
|
||||
int retVal = 1;
|
||||
retVal = ipfs_blockstore_put_node(node, fs_repo, bytes_written);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
// take the hash, base32 it, and send both to the datastore
|
||||
size_t fs_key_length = 100;
|
||||
unsigned char fs_key[fs_key_length];
|
||||
retVal = ipfs_datastore_helper_ds_key_from_binary(node->hash, node->hash_size, fs_key, fs_key_length, &fs_key_length);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
retVal = fs_repo->config->datastore->datastore_put(node->hash, node->hash_size, fs_key, fs_key_length, fs_repo->config->datastore);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ipfs_repo_fsrepo_node_read(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo) {
|
||||
int retVal = 0;
|
||||
|
||||
|
|
@ -569,3 +632,18 @@ int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, s
|
|||
return retVal;
|
||||
}
|
||||
|
||||
int ipfs_repo_fsrepo_unixfs_read(const unsigned char* hash, size_t hash_length, struct UnixFS** unix_fs, const struct FSRepo* fs_repo) {
|
||||
int retVal = 0;
|
||||
|
||||
// get the base32 hash from the database
|
||||
// We do this only to see if it is in the database
|
||||
size_t fs_key_length = 100;
|
||||
unsigned char fs_key[fs_key_length];
|
||||
retVal = fs_repo->config->datastore->datastore_get((const char*)hash, hash_length, fs_key, fs_key_length, &fs_key_length, fs_repo->config->datastore);
|
||||
if (retVal == 0) // maybe it doesn't exist?
|
||||
return 0;
|
||||
// now get the block from the blockstore
|
||||
retVal = ipfs_blockstore_get_unixfs(hash, hash_length, unix_fs, fs_repo);
|
||||
return retVal;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -42,8 +42,8 @@ int test_merkledag_get_data() {
|
|||
// create a node
|
||||
struct Node* node1;
|
||||
retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1);
|
||||
|
||||
retVal = ipfs_merkledag_add(node1, fs_repo);
|
||||
size_t bytes_written = 0;
|
||||
retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
|
||||
if (retVal == 0) {
|
||||
ipfs_node_free(node1);
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
|
|
@ -105,7 +105,8 @@ int test_merkledag_add_data() {
|
|||
struct Node* node1;
|
||||
retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1);
|
||||
|
||||
retVal = ipfs_merkledag_add(node1, fs_repo);
|
||||
size_t bytes_written = 0;
|
||||
retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
|
||||
if (retVal == 0) {
|
||||
ipfs_node_free(node1);
|
||||
return 0;
|
||||
|
|
@ -124,7 +125,7 @@ int test_merkledag_add_data() {
|
|||
// adding the same binary again should do nothing (the hash should be the same)
|
||||
struct Node* node2;
|
||||
retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node2);
|
||||
retVal = ipfs_merkledag_add(node2, fs_repo);
|
||||
retVal = ipfs_merkledag_add(node2, fs_repo, &bytes_written);
|
||||
if (retVal == 0) {
|
||||
ipfs_node_free(node1);
|
||||
ipfs_node_free(node2);
|
||||
|
|
@ -160,7 +161,7 @@ int test_merkledag_add_data() {
|
|||
struct Node* node3;
|
||||
retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node3);
|
||||
|
||||
retVal = ipfs_merkledag_add(node3, fs_repo);
|
||||
retVal = ipfs_merkledag_add(node3, fs_repo, &bytes_written);
|
||||
if (retVal == 0) {
|
||||
ipfs_node_free(node1);
|
||||
ipfs_node_free(node2);
|
||||
|
|
@ -207,7 +208,8 @@ int test_merkledag_add_node() {
|
|||
return 0;
|
||||
}
|
||||
|
||||
retVal = ipfs_merkledag_add(node1, fs_repo);
|
||||
size_t bytes_written = 0;
|
||||
retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
|
||||
if (retVal == 0) {
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
ipfs_node_free(node1);
|
||||
|
|
@ -250,7 +252,8 @@ int test_merkledag_add_node_with_links() {
|
|||
return 0;
|
||||
}
|
||||
|
||||
retVal = ipfs_merkledag_add(node1, fs_repo);
|
||||
size_t bytes_written = 0;
|
||||
retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
|
||||
if (retVal == 0) {
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
ipfs_node_free(node1);
|
||||
|
|
|
|||
|
|
@ -65,7 +65,6 @@ int test_unixfs_encode_smallfile() {
|
|||
memcpy(unixfs->bytes, bytes, 35);
|
||||
unixfs->bytes_size = 35;
|
||||
unixfs->data_type = UNIXFS_FILE;
|
||||
unixfs->file_size = 35;
|
||||
|
||||
size_t protobuf_size = 43;
|
||||
unsigned char protobuf[protobuf_size];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
CC = gcc
|
||||
CFLAGS = -O0 -I../include -I../../c-libp2p/include -I../../c-multihash/include -I../../c-multiaddr/include -I../../c-protobuf -Wall
|
||||
|
||||
ifdef DEBUG
|
||||
CFLAGS += -g3
|
||||
endif
|
||||
|
||||
LFLAGS =
|
||||
DEPS =
|
||||
OBJS = unixfs.o
|
||||
|
||||
%.o: %.c $(DEPS)
|
||||
$(CC) -c -o $@ $< $(CFLAGS)
|
||||
|
||||
all: $(OBJS)
|
||||
|
||||
clean:
|
||||
rm -f *.o
|
||||
|
|
@ -1,7 +1,10 @@
|
|||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "ipfs/unixfs/format.h"
|
||||
|
||||
|
||||
/**
|
||||
* Copy data into a new struct
|
||||
* @param type the type of data
|
||||
|
|
@ -12,7 +15,7 @@
|
|||
*/
|
||||
int ipfs_unixfs_format_new(enum UnixFSFormatType type, const unsigned char* data,
|
||||
size_t data_length, struct UnixFSData** result) {
|
||||
struct UnixFSData* new_struct = malloc(sizeof(struct UnixFSData));
|
||||
struct UnixFSData* new_struct = (struct UnixFSData*)malloc(sizeof(struct UnixFSData));
|
||||
if (new_struct == NULL)
|
||||
return 0;
|
||||
|
||||
|
|
@ -4,7 +4,9 @@
|
|||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "libp2p/crypto/sha256.h"
|
||||
#include "ipfs/unixfs/unixfs.h"
|
||||
#include "protobuf.h"
|
||||
#include "varint.h"
|
||||
|
|
@ -43,8 +45,9 @@ int ipfs_unixfs_new(struct UnixFS** obj) {
|
|||
(*obj)->bytes = 0;
|
||||
(*obj)->bytes_size = 0;
|
||||
(*obj)->data_type = UNIXFS_RAW;
|
||||
(*obj)->file_size = 0;
|
||||
(*obj)->block_size_head = NULL;
|
||||
(*obj)->hash = NULL;
|
||||
(*obj)->hash_length = 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
@ -56,6 +59,40 @@ int ipfs_unixfs_free(struct UnixFS* obj) {
|
|||
return 1;
|
||||
}
|
||||
|
||||
/***
|
||||
* Write data to data section of a UnixFS stuct. NOTE: this also calculates a sha256 hash
|
||||
* @param data the data to write
|
||||
* @param data_length the length of the data
|
||||
* @param unix_fs the struct to add to
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs) {
|
||||
|
||||
unix_fs->bytes_size = data_length;
|
||||
unix_fs->bytes = malloc(sizeof(unsigned char) * data_length);
|
||||
if ( unix_fs->bytes == NULL) {
|
||||
return 0;
|
||||
}
|
||||
memcpy( unix_fs->bytes, data, data_length);
|
||||
|
||||
// now compute the hash
|
||||
unix_fs->hash_length = 32;
|
||||
unix_fs->hash = (unsigned char*)malloc(unix_fs->hash_length);
|
||||
if (unix_fs->hash == NULL) {
|
||||
free(unix_fs->bytes);
|
||||
return 0;
|
||||
}
|
||||
if (libp2p_crypto_hashing_sha256(data, data_length, &unix_fs->hash[0]) == 0) {
|
||||
free(unix_fs->bytes);
|
||||
free(unix_fs->hash);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Protobuf functions
|
||||
*/
|
||||
|
|
@ -68,7 +105,7 @@ enum WireType ipfs_unixfs_message_fields[] = { WIRETYPE_VARINT, WIRETYPE_LENGTH_
|
|||
* @param obj what will be encoded
|
||||
* @returns the size of the buffer necessary to encode the object
|
||||
*/
|
||||
size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj) {
|
||||
size_t ipfs_unixfs_protobuf_encode_size(const struct UnixFS* obj) {
|
||||
size_t sz = 0;
|
||||
// bytes
|
||||
sz += obj->bytes_size + 11;
|
||||
|
|
@ -111,8 +148,8 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou
|
|||
*bytes_written += bytes_used;
|
||||
}
|
||||
// file size (optional)
|
||||
if (incoming->file_size > 0) {
|
||||
retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
|
||||
if (incoming->data_type == UNIXFS_FILE && incoming->bytes_size > 0) {
|
||||
retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
*bytes_written += bytes_used;
|
||||
|
|
@ -168,7 +205,7 @@ int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, s
|
|||
pos += bytes_read;
|
||||
break;
|
||||
case (3): // file size
|
||||
result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
|
||||
result->bytes_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
|
||||
pos += bytes_read;
|
||||
break;
|
||||
case (4): { // block sizes (linked list from varint)
|
||||
|
|
|
|||
Loading…
Reference in New Issue