More work on persisting data to disk.
Blockstore now storing the data, whereas datastore is storing the key and filename. The key should be the multihash (currently the sha256, not the multihash), and the value is the filename (base32).
This commit is contained in:
jmjatlanta
parent
88d177cf4a
commit
033dd767b4
20 changed files with 564 additions and 57 deletions
117
blocks/block.c
117
blocks/block.c
|
|
@ -9,6 +9,100 @@
|
|||
#include "ipfs/blocks/block.h"
|
||||
#include "ipfs/cid/cid.h"
|
||||
|
||||
/***
|
||||
* The protobuf functions
|
||||
*/
|
||||
// protobuf fields: data & data_length cid
|
||||
enum WireType ipfs_block_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED};
|
||||
|
||||
/**
|
||||
* Determine the approximate size of an encoded block
|
||||
* @param block the block to measure
|
||||
* @returns the approximate size needed to encode the protobuf
|
||||
*/
|
||||
size_t ipfs_blocks_block_protobuf_encode_size(const struct Block* block) {
|
||||
return 22 + ipfs_cid_protobuf_encode_size(block->cid) + block->data_length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode the Block into protobuf format
|
||||
* @param block the block to encode
|
||||
* @param buffer the buffer to fill
|
||||
* @param max_buffer_size the max size of the buffer
|
||||
* @param bytes_written the number of bytes used
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blocks_block_protobuf_encode(const struct Block* block, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
|
||||
// data & data_size
|
||||
size_t bytes_used = 0;
|
||||
*bytes_written = 0;
|
||||
int retVal = 0;
|
||||
retVal = protobuf_encode_length_delimited(1, ipfs_block_message_fields[0], (char*)block->data, block->data_length, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
|
||||
*bytes_written += bytes_used;
|
||||
// cid
|
||||
size_t cid_size = ipfs_cid_protobuf_encode_size(block->cid);
|
||||
unsigned char cid[cid_size];
|
||||
retVal = ipfs_cid_protobuf_encode(block->cid, cid, cid_size, &cid_size);
|
||||
retVal = protobuf_encode_length_delimited(2, ipfs_block_message_fields[1], (char*)cid, cid_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
|
||||
*bytes_written += bytes_used;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/***
|
||||
* Decode from a protobuf stream into a Block struct
|
||||
* @param buffer the buffer to pull from
|
||||
* @param buffer_length the length of the buffer
|
||||
* @param block the block to fill
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blocks_block_protobuf_decode(const unsigned char* buffer, const size_t buffer_length, struct Block** block) {
|
||||
size_t pos = 0;
|
||||
int retVal = 0;
|
||||
unsigned char* temp_buffer = NULL;
|
||||
size_t temp_size;
|
||||
|
||||
if (ipfs_blocks_block_new(block) == 0)
|
||||
goto exit;
|
||||
|
||||
while(pos < buffer_length) {
|
||||
size_t bytes_read = 0;
|
||||
int field_no;
|
||||
enum WireType field_type;
|
||||
if (protobuf_decode_field_and_type(&buffer[pos], buffer_length, &field_no, &field_type, &bytes_read) == 0) {
|
||||
goto exit;
|
||||
}
|
||||
pos += bytes_read;
|
||||
switch(field_no) {
|
||||
case (1): // data
|
||||
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*block)->data), &((*block)->data_length), &bytes_read) == 0)
|
||||
goto exit;
|
||||
pos += bytes_read;
|
||||
break;
|
||||
case (2): // cid
|
||||
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0)
|
||||
goto exit;
|
||||
pos += bytes_read;
|
||||
if (ipfs_cid_protobuf_decode(temp_buffer, temp_size, &((*block)->cid)) == 0)
|
||||
goto exit;
|
||||
free(temp_buffer);
|
||||
temp_buffer = NULL;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
retVal = 1;
|
||||
|
||||
exit:
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(*block);
|
||||
}
|
||||
if (temp_buffer != NULL)
|
||||
free(temp_buffer);
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
|
||||
/***
|
||||
* Create a new block based on the incoming data
|
||||
* @param data the data to base the block on
|
||||
|
|
@ -16,35 +110,38 @@
|
|||
* @param block a pointer to the struct Block that will be created
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block) {
|
||||
int ipfs_blocks_block_new(struct Block** block) {
|
||||
|
||||
// allocate memory for structure
|
||||
(*block) = (struct Block*)malloc(sizeof(struct Block));
|
||||
if ((*block) == NULL)
|
||||
return 0;
|
||||
(*block)->data = NULL;
|
||||
(*block)->data_length = 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, struct Block* block) {
|
||||
// cid
|
||||
unsigned char hash[32];
|
||||
if (libp2p_crypto_hashing_sha256(data, data_size, &hash[0]) == 0) {
|
||||
free(*block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &((*block)->cid)) == 0) {
|
||||
free(*block);
|
||||
if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &(block->cid)) == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
(*block)->data_length = data_size;
|
||||
block->data_length = data_size;
|
||||
|
||||
(*block)->data = malloc(sizeof(unsigned char) * data_size);
|
||||
if ( (*block)->data == NULL) {
|
||||
ipfs_cid_free((*block)->cid);
|
||||
free(*block);
|
||||
block->data = malloc(sizeof(unsigned char) * data_size);
|
||||
if ( block->data == NULL) {
|
||||
ipfs_cid_free(block->cid);
|
||||
return 0;
|
||||
}
|
||||
|
||||
memcpy( (*block)->data, data, data_size);
|
||||
memcpy( block->data, data, data_size);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include "ipfs/blocks/block.h"
|
||||
#include "ipfs/datastore/ds_helper.h"
|
||||
#include "ipfs/repo/fsrepo/fs_repo.h"
|
||||
#include "ipfs/os/utils.h"
|
||||
|
||||
/**
|
||||
* Delete a block based on its Cid
|
||||
|
|
@ -25,14 +26,59 @@ int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
unsigned char* ipfs_blockstore_cid_to_base32(const struct Cid* cid) {
|
||||
size_t key_length = libp2p_crypto_encoding_base32_encode_size(cid->hash_length);
|
||||
unsigned char* buffer = (unsigned char*)malloc(key_length + 1);
|
||||
if (buffer == NULL)
|
||||
return NULL;
|
||||
int retVal = ipfs_datastore_helper_ds_key_from_binary(cid->hash, cid->hash_length, &buffer[0], key_length, &key_length);
|
||||
if (retVal == 0) {
|
||||
free(buffer);
|
||||
return NULL;
|
||||
}
|
||||
buffer[key_length] = 0;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
char* ipfs_blockstore_path_get(const struct FSRepo* fs_repo, const char* filename) {
|
||||
int filepath_size = strlen(fs_repo->path) + 12;
|
||||
char filepath[filepath_size];
|
||||
int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", filepath, filepath_size);
|
||||
if (retVal == 0) {
|
||||
free(filepath);
|
||||
return 0;
|
||||
}
|
||||
int complete_filename_size = strlen(filepath) + strlen(filename) + 2;
|
||||
char* complete_filename = (char*)malloc(complete_filename_size);
|
||||
retVal = os_utils_filepath_join(filepath, filename, complete_filename, complete_filename_size);
|
||||
return complete_filename;
|
||||
}
|
||||
|
||||
/***
|
||||
* Find a block based on its Cid
|
||||
* @param cid the Cid to look for
|
||||
* @param block where to put the data to be returned
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo) {
|
||||
return 0;
|
||||
int ipfs_blockstore_get(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo) {
|
||||
// get datastore key, which is a base32 key of the multihash
|
||||
unsigned char* key = ipfs_blockstore_cid_to_base32(cid);
|
||||
|
||||
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
|
||||
|
||||
size_t file_size = os_utils_file_size(filename);
|
||||
unsigned char buffer[file_size];
|
||||
|
||||
FILE* file = fopen(filename, "rb");
|
||||
size_t bytes_read = fread(buffer, 1, file_size, file);
|
||||
fclose(file);
|
||||
|
||||
int retVal = ipfs_blocks_block_protobuf_decode(buffer, bytes_read, block);
|
||||
|
||||
free(key);
|
||||
free(filename);
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
/***
|
||||
|
|
@ -42,14 +88,46 @@ int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_
|
|||
*/
|
||||
int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) {
|
||||
// from blockstore.go line 118
|
||||
// Get Datastore key, which is a base32 key of the binary,
|
||||
size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length);
|
||||
unsigned char key[key_length];
|
||||
int retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
int retVal = 0;
|
||||
|
||||
// send to Put with key
|
||||
fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
|
||||
// Get Datastore key, which is a base32 key of the multihash,
|
||||
unsigned char* key = ipfs_blockstore_cid_to_base32(block->cid);
|
||||
if (key == NULL) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//TODO: put this in subdirectories
|
||||
|
||||
// turn the block into a binary array
|
||||
size_t protobuf_len = ipfs_blocks_block_protobuf_encode_size(block);
|
||||
unsigned char protobuf[protobuf_len];
|
||||
retVal = ipfs_blocks_block_protobuf_encode(block, protobuf, protobuf_len, &protobuf_len);
|
||||
if (retVal == 0) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// now write byte array to file
|
||||
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
|
||||
if (filename == NULL) {
|
||||
free(key);
|
||||
return 0;
|
||||
}
|
||||
|
||||
FILE* file = fopen(filename, "wb");
|
||||
int bytes_written = fwrite(protobuf, 1, protobuf_len, file);
|
||||
fclose(file);
|
||||
if (bytes_written != protobuf_len) {
|
||||
free(key);
|
||||
free(filename);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// send to Put with key (this is now done separately)
|
||||
//fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
|
||||
|
||||
free(key);
|
||||
free(filename);
|
||||
return 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
/**
|
||||
* Some code to help with the datastore / blockstore interface
|
||||
* NOTE: the datastore stores things under a multihash key
|
||||
*/
|
||||
#include "libp2p/crypto/encoding/base32.h"
|
||||
#include "ipfs/datastore/ds_helper.h"
|
||||
/**
|
||||
* Generate a key based on the passed in binary_array
|
||||
* Generate a base32 key based on the passed in binary_array (which is normally a multihash)
|
||||
* @param binary_array what to base the key on
|
||||
* @param array_length the size of the binary array
|
||||
* @param results where the key will be put
|
||||
|
|
@ -30,7 +31,7 @@ int ipfs_datastore_helper_ds_key_from_binary(unsigned char* binary_array, size_t
|
|||
}
|
||||
|
||||
/**
|
||||
* Generate a binary array based on the passed in datastore key
|
||||
* Generate a binary array (normally a multihash) based on the passed in datastore key
|
||||
* @param ds_key the base32 encoded key
|
||||
* @param key_length the length of the base32 "string"
|
||||
* @param binary_array where to put the decoded value
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#include <stdio.h>
|
||||
|
||||
#include "ipfs/importer/importer.h"
|
||||
#include "ipfs/merkledag/merkledag.h"
|
||||
|
||||
#define MAX_DATA_SIZE 262144 // 1024 * 256;
|
||||
|
||||
|
|
@ -14,26 +15,27 @@
|
|||
* @param node the node to add to
|
||||
* @returns number of bytes read
|
||||
*/
|
||||
size_t ipfs_import_chunk(FILE* file, struct Node* node) {
|
||||
size_t ipfs_import_chunk(FILE* file, struct Node* node, struct FSRepo* fs_repo) {
|
||||
unsigned char buffer[MAX_DATA_SIZE];
|
||||
size_t bytes_read = fread(buffer, MAX_DATA_SIZE, 1, file);
|
||||
size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file);
|
||||
if (node->data_size == 0) {
|
||||
ipfs_node_set_data(node, buffer, bytes_read);
|
||||
if (bytes_read != MAX_DATA_SIZE) {
|
||||
// persist
|
||||
}
|
||||
} else {
|
||||
// create a new node, and link to the parent
|
||||
struct Node* new_node = NULL;
|
||||
ipfs_node_new_from_data(buffer, bytes_read, &new_node);
|
||||
// persist
|
||||
|
||||
ipfs_merkledag_add(new_node, fs_repo);
|
||||
// put link in node
|
||||
struct NodeLink* new_link = NULL;
|
||||
ipfs_node_link_new("", new_node->cached->hash, &new_link);
|
||||
ipfs_node_add_link(node, new_link);
|
||||
ipfs_node_free(new_node);
|
||||
}
|
||||
if (bytes_read != MAX_DATA_SIZE) {
|
||||
// persist the main node
|
||||
ipfs_merkledag_add(node, fs_repo);
|
||||
}
|
||||
return bytes_read;
|
||||
}
|
||||
|
||||
|
|
@ -43,7 +45,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* node) {
|
|||
* @param node the root node (could have links to others)
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_import_file(const char* fileName, struct Node** node) {
|
||||
int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo) {
|
||||
int retVal = 1;
|
||||
int bytes_read = MAX_DATA_SIZE;
|
||||
|
||||
|
|
@ -54,7 +56,7 @@ int ipfs_import_file(const char* fileName, struct Node** node) {
|
|||
|
||||
// add all nodes
|
||||
while ( bytes_read == MAX_DATA_SIZE) {
|
||||
bytes_read = ipfs_import_chunk(file, *node);
|
||||
bytes_read = ipfs_import_chunk(file, *node, fs_repo);
|
||||
}
|
||||
fclose(file);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
/***
|
||||
* IPFS has the notion of storage blocks.
|
||||
* Raw data with a multihash key (the Cid)
|
||||
*/
|
||||
|
||||
#ifndef __IPFS_BLOCKS_BLOCK_H__
|
||||
|
|
@ -20,7 +21,9 @@ struct Block {
|
|||
* @param block a pointer to the struct Block that will be created
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block);
|
||||
int ipfs_blocks_block_new(struct Block** block);
|
||||
|
||||
int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, struct Block* block);
|
||||
|
||||
/***
|
||||
* Free resources used by the creation of a block
|
||||
|
|
@ -29,4 +32,30 @@ int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Bl
|
|||
*/
|
||||
int ipfs_blocks_block_free(struct Block* block);
|
||||
|
||||
/**
|
||||
* Determine the approximate size of an encoded block
|
||||
* @param block the block to measure
|
||||
* @returns the approximate size needed to encode the protobuf
|
||||
*/
|
||||
size_t ipfs_blocks_block_protobuf_encode_size(const struct Block* block);
|
||||
|
||||
/**
|
||||
* Encode the Block into protobuf format
|
||||
* @param block the block to encode
|
||||
* @param buffer the buffer to fill
|
||||
* @param max_buffer_size the max size of the buffer
|
||||
* @param bytes_written the number of bytes used
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blocks_block_protobuf_encode(const struct Block* block, unsigned char* buffer, size_t max_buffer_size, size_t* bytes_written);
|
||||
|
||||
/***
|
||||
* Decode from a protobuf stream into a Block struct
|
||||
* @param buffer the buffer to pull from
|
||||
* @param buffer_length the length of the buffer
|
||||
* @param block the block to fill
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blocks_block_protobuf_decode(const unsigned char* buffer, const size_t buffer_length, struct Block** block);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -3,7 +3,10 @@
|
|||
*/
|
||||
|
||||
#ifndef __IPFS_BLOCKS_BLOCKSTORE_H__
|
||||
#ifndef __IPFS_BLOCKS_BLOCKSTORE_H__
|
||||
#define __IPFS_BLOCKS_BLOCKSTORE_H__
|
||||
|
||||
#include "ipfs/cid/cid.h"
|
||||
#include "ipfs/repo/fsrepo/fs_repo.h"
|
||||
|
||||
/**
|
||||
* Delete a block based on its Cid
|
||||
|
|
@ -25,14 +28,14 @@ int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo);
|
|||
* @param block where to put the data to be returned
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo);
|
||||
int ipfs_blockstore_get(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo);
|
||||
|
||||
/***
|
||||
* Put a block in the blockstore
|
||||
* @param block the block to store
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo);
|
||||
int ipfs_blockstore_put(struct Block* block, const struct FSRepo* fs_repo);
|
||||
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#define __IPFS_IMPORTER_IMPORTER_H__
|
||||
|
||||
#include "ipfs/node/node.h"
|
||||
#include "ipfs/repo/fsrepo/fs_repo.h"
|
||||
|
||||
/**
|
||||
* Creates a node based on an incoming file
|
||||
|
|
@ -9,6 +10,6 @@
|
|||
* @param node the root node (could have links to others)
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_import_file(const char* fileName, struct Node** node);
|
||||
int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo);
|
||||
|
||||
#endif /* INCLUDE_IPFS_IMPORTER_IMPORTER_H_ */
|
||||
|
|
|
|||
|
|
@ -64,4 +64,13 @@ int ipfs_repo_fsrepo_free(struct FSRepo* config);
|
|||
*/
|
||||
int ipfs_repo_fsrepo_init(struct FSRepo* config);
|
||||
|
||||
/***
|
||||
* Write a block to the datastore and blockstore
|
||||
* @param block the block to write
|
||||
* @param fs_repo the repo to write to
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo);
|
||||
int ipfs_repo_fsrepo_block_read(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo);
|
||||
|
||||
#endif /* fs_repo_h */
|
||||
|
|
|
|||
|
|
@ -21,9 +21,11 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
|
|||
|
||||
// turn the node into a block
|
||||
struct Block* block;
|
||||
ipfs_blocks_block_new(protobuf, bytes_written, &block);
|
||||
ipfs_blocks_block_new(&block);
|
||||
ipfs_blocks_block_add_data(protobuf, bytes_written, block);
|
||||
|
||||
int retVal = fs_repo->config->datastore->datastore_put_block(block, fs_repo->config->datastore);
|
||||
// write to block store & datastore
|
||||
int retVal = ipfs_repo_fsrepo_block_write(block, fs_repo);
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
|
|
@ -45,17 +47,25 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
|
|||
*/
|
||||
int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo) {
|
||||
int retVal = 1;
|
||||
|
||||
// look for the block
|
||||
struct Block* block;
|
||||
retVal = fs_repo->config->datastore->datastore_get_block(cid, &block, fs_repo->config->datastore);
|
||||
size_t key_length = 100;
|
||||
unsigned char key[key_length];
|
||||
|
||||
// look for the node in the datastore. If it is not there, it is not a node.
|
||||
// If it exists, it is only a block.
|
||||
retVal = fs_repo->config->datastore->datastore_get((char*)cid->hash, cid->hash_length, key, key_length, &key_length, fs_repo->config->datastore);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
|
||||
// we have the block. Fill the node
|
||||
// we have the record from the db. Go get the block from the blockstore
|
||||
retVal = ipfs_repo_fsrepo_block_read(cid, &block, fs_repo);
|
||||
|
||||
// now convert the block into a node
|
||||
ipfs_node_protobuf_decode(block->data, block->data_length, node);
|
||||
// doesn't decode do this?
|
||||
ipfs_node_set_cached(*node, cid);
|
||||
|
||||
// free resources
|
||||
ipfs_blocks_block_free(block);
|
||||
|
||||
return retVal;
|
||||
|
|
|
|||
|
|
@ -295,6 +295,7 @@ int ipfs_node_new(struct Node** node)
|
|||
return 0;
|
||||
(*node)->cached = NULL;
|
||||
(*node)->data = NULL;
|
||||
(*node)->data_size = 0;
|
||||
(*node)->encoded = NULL;
|
||||
(*node)->head_link = NULL;
|
||||
return 1;
|
||||
|
|
|
|||
|
|
@ -62,11 +62,14 @@ int os_utils_directory_writeable(const char* path) {
|
|||
}
|
||||
|
||||
int os_utils_file_size(const char* path) {
|
||||
size_t file_size = 0;
|
||||
// open file
|
||||
FILE* in_file = fopen(path, "r");
|
||||
if (in_file == NULL)
|
||||
return 0;
|
||||
// determine size
|
||||
fseek(in_file, 0L, SEEK_END);
|
||||
size_t file_size = ftell(in_file);
|
||||
file_size = ftell(in_file);
|
||||
fclose(in_file);
|
||||
return file_size;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
#include <stdio.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include "ipfs/blocks/blockstore.h"
|
||||
#include "libp2p/crypto/encoding/base64.h"
|
||||
#include "ipfs/datastore/ds_helper.h"
|
||||
#include "ipfs/repo/config/datastore.h"
|
||||
#include "ipfs/repo/fsrepo/fs_repo.h"
|
||||
#include "ipfs/os/utils.h"
|
||||
|
|
@ -460,6 +463,18 @@ int ipfs_repo_fsrepo_datastore_init(struct FSRepo* fs_repo) {
|
|||
return repo_fsrepo_lmdb_cast(fs_repo->config->datastore);
|
||||
}
|
||||
|
||||
int ipfs_repo_fsrepo_blockstore_init(const struct FSRepo* fs_repo) {
|
||||
size_t full_path_size = strlen(fs_repo->path) + 15;
|
||||
char full_path[full_path_size];
|
||||
int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", full_path, full_path_size);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
|
||||
if (mkdir(full_path, S_IRWXU) != 0)
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes a new FSRepo at the given path with the provided config
|
||||
* @param path the path to use
|
||||
|
|
@ -477,11 +492,14 @@ int ipfs_repo_fsrepo_init(struct FSRepo* repo) {
|
|||
if (retVal == 0)
|
||||
return 0;
|
||||
|
||||
// TODO: Implement this method
|
||||
retVal = ipfs_repo_fsrepo_datastore_init(repo);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
|
||||
retVal = ipfs_repo_fsrepo_blockstore_init(repo);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
|
||||
// write the version to a file for migrations (see repo/fsrepo/migrations/mfsr.go)
|
||||
//TODO: mfsr.RepoPath(repo_path).WriteVersion(RepoVersion)
|
||||
return 1;
|
||||
|
|
@ -508,3 +526,46 @@ int fs_repo_write_config_file(char* path, struct RepoConfig* config) {
|
|||
return retVal;
|
||||
}
|
||||
|
||||
/***
|
||||
* Write a block to the datastore and blockstore
|
||||
* @param block the block to write
|
||||
* @param fs_repo the repo to write to
|
||||
* @returns true(1) on success
|
||||
*/
|
||||
int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo) {
|
||||
/**
|
||||
* What is put in the blockstore is the block.
|
||||
* What is put in the datastore is the multihash (the Cid) as the key,
|
||||
* and the base32 encoded multihash as the value.
|
||||
*/
|
||||
int retVal = 1;
|
||||
retVal = ipfs_blockstore_put(block, fs_repo);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
// take the cid, base32 it, and send both to the datastore
|
||||
size_t fs_key_length = 100;
|
||||
unsigned char fs_key[fs_key_length];
|
||||
retVal = ipfs_datastore_helper_ds_key_from_binary(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, &fs_key_length);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
retVal = fs_repo->config->datastore->datastore_put(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, fs_repo->config->datastore);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ipfs_repo_fsrepo_block_read(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo) {
|
||||
int retVal = 0;
|
||||
|
||||
// get the base32 hash from the database
|
||||
// We do this only to see if it is in the database
|
||||
size_t fs_key_length = 100;
|
||||
unsigned char fs_key[fs_key_length];
|
||||
retVal = fs_repo->config->datastore->datastore_get((char*)cid->hash, cid->hash_length, fs_key, fs_key_length, &fs_key_length, fs_repo->config->datastore);
|
||||
if (retVal == 0) // maybe it doesn't exist?
|
||||
return 0;
|
||||
// now get the block from the blockstore
|
||||
retVal = ipfs_blockstore_get(cid, block, fs_repo);
|
||||
return retVal;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
/***
|
||||
* Here are the wrappers for the lightning database
|
||||
* NOTE: In this implementation, the database will contain the base32 encoded value
|
||||
* of the multihash key if the file exists on disk.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
|
|
@ -43,12 +45,13 @@ int repo_fsrepo_lmdb_get_block(const struct Cid* cid, struct Block** block, cons
|
|||
}
|
||||
|
||||
// now copy the data
|
||||
retVal = ipfs_blocks_block_new(db_value.mv_data, db_value.mv_size, block);
|
||||
retVal = ipfs_blocks_block_new(block);
|
||||
if (retVal == 0) {
|
||||
mdb_dbi_close(mdb_env, mdb_dbi);
|
||||
mdb_txn_commit(mdb_txn);
|
||||
return 0;
|
||||
}
|
||||
retVal = ipfs_blocks_block_add_data(db_value.mv_data, db_value.mv_size, *block);
|
||||
|
||||
// clean up
|
||||
mdb_dbi_close(mdb_env, mdb_dbi);
|
||||
|
|
|
|||
77
test/node/test_importer.h
Normal file
77
test/node/test_importer.h
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
#include <stdio.h>
|
||||
|
||||
#include "ipfs/importer/importer.h"
|
||||
#include "ipfs/merkledag/merkledag.h"
|
||||
|
||||
/***
|
||||
* Helper to create a test file in the OS
|
||||
*/
|
||||
int create_file(const char* fileName, unsigned char* bytes, size_t num_bytes) {
|
||||
FILE* file = fopen(fileName, "wb");
|
||||
fwrite(bytes, num_bytes, 1, file);
|
||||
fclose(file);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int create_bytes(unsigned char* buffer, size_t num_bytes) {
|
||||
int counter = 0;
|
||||
|
||||
for(int i = 0; i < num_bytes; i++) {
|
||||
buffer[i] = counter++;
|
||||
if (counter > 15)
|
||||
counter = 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int test_import_small_file() {
|
||||
size_t bytes_size = 1000;
|
||||
unsigned char file_bytes[bytes_size];
|
||||
const char* fileName = "/tmp/test_import_small.tmp";
|
||||
|
||||
// create the necessary file
|
||||
create_bytes(file_bytes, bytes_size);
|
||||
create_file(fileName, file_bytes, bytes_size);
|
||||
|
||||
// get the repo
|
||||
drop_and_build_repository("/tmp/.ipfs");
|
||||
struct FSRepo* fs_repo;
|
||||
ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, &fs_repo);
|
||||
ipfs_repo_fsrepo_open(fs_repo);
|
||||
|
||||
// write to ipfs
|
||||
struct Node* write_node;
|
||||
if (ipfs_import_file(fileName, &write_node, fs_repo) == 0) {
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// make sure all went okay
|
||||
struct Node* read_node;
|
||||
if (ipfs_merkledag_get(write_node->cached, &read_node, fs_repo) == 0) {
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
ipfs_node_free(write_node);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// compare data
|
||||
if (write_node->data_size != bytes_size || write_node->data_size != read_node->data_size) {
|
||||
printf("Data size of nodes are not equal or are incorrect. Should be %lu but are %lu\n", write_node->data_size, read_node->data_size);
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
ipfs_node_free(write_node);
|
||||
ipfs_node_free(read_node);
|
||||
return 0;
|
||||
}
|
||||
|
||||
for(int i = 0; i < bytes_size; i++) {
|
||||
if (write_node->data[i] != read_node->data[i]) {
|
||||
printf("Data within node is different at position %d\n", i);
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
ipfs_node_free(write_node);
|
||||
ipfs_node_free(read_node);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
#include "ipfs/repo/fsrepo/fs_repo.h"
|
||||
#include "../test_helper.h"
|
||||
|
||||
int test_repo_fsrepo_open_config() {
|
||||
struct FSRepo* fs_repo = NULL;
|
||||
|
|
@ -24,3 +25,73 @@ int test_repo_fsrepo_open_config() {
|
|||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int test_repo_fsrepo_write_read_block() {
|
||||
struct Block* block = NULL;
|
||||
struct FSRepo* fs_repo = NULL;
|
||||
int retVal = 0;
|
||||
|
||||
// freshen the repository
|
||||
retVal = drop_build_and_open_repo("/tmp/.ipfs", &fs_repo);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
|
||||
// make some data
|
||||
size_t data_size = 100;
|
||||
unsigned char data[data_size];
|
||||
|
||||
int counter = 0;
|
||||
for(int i = 0; i < data_size; i++) {
|
||||
data[i] = counter++;
|
||||
if (counter > 15)
|
||||
counter = 0;
|
||||
}
|
||||
|
||||
// create and write the block
|
||||
retVal = ipfs_blocks_block_new(&block);
|
||||
if (retVal == 0) {
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
return 0;
|
||||
}
|
||||
retVal = ipfs_blocks_block_add_data(data, data_size, block);
|
||||
if (retVal == 0) {
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
retVal = ipfs_repo_fsrepo_block_write(block, fs_repo);
|
||||
if (retVal == 0) {
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// retrieve the block
|
||||
struct Block* results;
|
||||
retVal = ipfs_repo_fsrepo_block_read(block->cid, &results, fs_repo);
|
||||
if (retVal == 0) {
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// compare the two blocks
|
||||
retVal = 1;
|
||||
if (block->data_length != results->data_length || block->data_length != data_size) {
|
||||
printf("block data is of different length: %lu vs %lu\n", results->data_length, block->data_length);
|
||||
retVal = 0;
|
||||
}
|
||||
|
||||
for(size_t i = 0; i < block->data_length; i++) {
|
||||
if (block->data[i] != results->data[i]) {
|
||||
printf("Data is different at position %lu. Should be %02x but is %02x\n", i, block->data[i], results->data[i]);
|
||||
retVal = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ipfs_repo_fsrepo_free(fs_repo);
|
||||
ipfs_blocks_block_free(block);
|
||||
ipfs_blocks_block_free(results);
|
||||
return retVal;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,31 +4,49 @@ int test_blocks_new() {
|
|||
const unsigned char* input = (const unsigned char*)"Hello, World!";
|
||||
int retVal = 0;
|
||||
struct Block* block;
|
||||
retVal = ipfs_blocks_block_new(input, strlen((const char*)input) + 1, &block);
|
||||
retVal = ipfs_blocks_block_new(&block);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
|
||||
retVal = ipfs_blocks_block_add_data(input, strlen((const char*)input) + 1, block);
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// now examine the block
|
||||
if (strcmp((const char*)block->data, (const char*)input) != 0)
|
||||
if (strcmp((const char*)block->data, (const char*)input) != 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (block->data_length != strlen((const char*)input) + 1)
|
||||
if (block->data_length != strlen((const char*)input) + 1) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (block->cid->codec != CID_PROTOBUF)
|
||||
if (block->cid->codec != CID_PROTOBUF) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (block->cid->version != 0)
|
||||
if (block->cid->version != 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (block->cid->hash_length != 32)
|
||||
if (block->cid->hash_length != 32) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
unsigned char result_hash[32] = {33, 153, 66, 187, 124, 250, 87, 12, 12, 73, 43, 247, 175, 153, 10, 51, 192, 195, 218, 69, 220, 170, 105, 179, 195, 0, 203, 213, 172, 3, 244, 10 };
|
||||
for(int i = 0; i < 32; i++) {
|
||||
if (block->cid->hash[i] != result_hash[i])
|
||||
if (block->cid->hash[i] != result_hash[i]) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
retVal = ipfs_blocks_block_free(block);
|
||||
|
||||
|
|
|
|||
|
|
@ -23,30 +23,43 @@ int test_ipfs_datastore_put() {
|
|||
return 0;
|
||||
|
||||
// build the block
|
||||
retVal = ipfs_blocks_block_new(input, strlen((char*)input), &block);
|
||||
retVal = ipfs_blocks_block_new(&block);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
|
||||
retVal = ipfs_blocks_block_add_data(input, strlen((char*)input), block);
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// generate the key
|
||||
size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length);
|
||||
unsigned char key[key_length];
|
||||
retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length);
|
||||
if (retVal == 0)
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// open the repository
|
||||
struct FSRepo* fs_repo;
|
||||
retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, &fs_repo);
|
||||
if (retVal == 0)
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
retVal = ipfs_repo_fsrepo_open(fs_repo);
|
||||
if (retVal == 0)
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
|
||||
}
|
||||
// send to Put with key
|
||||
retVal = fs_repo->config->datastore->datastore_put((const unsigned char*)key, key_length, block->data, block->data_length, fs_repo->config->datastore);
|
||||
if (retVal == 0)
|
||||
if (retVal == 0) {
|
||||
ipfs_blocks_block_free(block);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// save the block
|
||||
|
||||
|
|
|
|||
|
|
@ -122,3 +122,22 @@ int make_ipfs_repository(const char* path) {
|
|||
int drop_and_build_repository(const char* path) {
|
||||
return make_ipfs_repository(path);
|
||||
}
|
||||
|
||||
|
||||
int drop_build_and_open_repo(const char* path, struct FSRepo** fs_repo) {
|
||||
int retVal = 0;
|
||||
|
||||
retVal = drop_and_build_repository("/tmp/.ipfs");
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, fs_repo);
|
||||
if (retVal == 0)
|
||||
return 0;
|
||||
retVal = ipfs_repo_fsrepo_open(*fs_repo);
|
||||
if (retVal == 0) {
|
||||
free(*fs_repo);
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,11 @@
|
|||
/**
|
||||
* Helpers for testing
|
||||
*/
|
||||
|
||||
/**
|
||||
* Create a new repository in the directory, erasing old one
|
||||
* NOTE: base directory must already exist
|
||||
*/
|
||||
int drop_and_build_repository(const char* dir);
|
||||
|
||||
int drop_build_and_open_repo(const char* path, struct FSRepo** fs_repo);
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
#include "flatfs/test_flatfs.h"
|
||||
#include "merkledag/test_merkledag.h"
|
||||
#include "node/test_node.h"
|
||||
#include "node/test_importer.h"
|
||||
#include "repo/test_repo_bootstrap_peers.h"
|
||||
#include "repo/test_repo_config.h"
|
||||
#include "repo/test_repo_fsrepo.h"
|
||||
|
|
@ -31,7 +32,9 @@ const char* names[] = {
|
|||
"test_repo_config_write",
|
||||
"test_repo_config_identity_new",
|
||||
"test_repo_config_identity_private_key",
|
||||
"test_repo_fsrepo_write_read_block",
|
||||
"test_get_init_command",
|
||||
"test_import_small_file",
|
||||
"test_repo_fsrepo_open_config",
|
||||
"test_flatfs_get_directory",
|
||||
"test_flatfs_get_filename",
|
||||
|
|
@ -58,7 +61,9 @@ int (*funcs[])(void) = {
|
|||
test_repo_config_write,
|
||||
test_repo_config_identity_new,
|
||||
test_repo_config_identity_private_key,
|
||||
test_repo_fsrepo_write_read_block,
|
||||
test_get_init_command,
|
||||
test_import_small_file,
|
||||
test_repo_fsrepo_open_config,
|
||||
test_flatfs_get_directory,
|
||||
test_flatfs_get_filename,
|
||||
|
|
|
|||
Loading…
Reference in a new issue