agorise/c-ipfs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

More work on persisting data to disk.

Browse source 
Blockstore now storing the data, whereas datastore is storing the key
and filename. The key should be the multihash (currently the sha256, not
the multihash), and the value is the filename (base32).
This commit is contained in:
jmjatlanta 2016-12-14 12:07:43 -05:00
parent 88d177cf4a
commit 033dd767b4
20 changed files with 564 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

117
blocks/block.c
View file

@ -9,6 +9,100 @@
#include "ipfs/blocks/block.h"
#include "ipfs/cid/cid.h"
/***
* The protobuf functions
*/
// protobuf fields: data & data_length cid
enum WireType ipfs_block_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED};
/**
* Determine the approximate size of an encoded block
* @param block the block to measure
* @returns the approximate size needed to encode the protobuf
*/
size_t ipfs_blocks_block_protobuf_encode_size(const struct Block* block) {
return 22 + ipfs_cid_protobuf_encode_size(block->cid) + block->data_length;
}
/**
* Encode the Block into protobuf format
* @param block the block to encode
* @param buffer the buffer to fill
* @param max_buffer_size the max size of the buffer
* @param bytes_written the number of bytes used
* @returns true(1) on success
*/
int ipfs_blocks_block_protobuf_encode(const struct Block* block, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
// data & data_size
size_t bytes_used = 0;
*bytes_written = 0;
int retVal = 0;
retVal = protobuf_encode_length_delimited(1, ipfs_block_message_fields[0], (char*)block->data, block->data_length, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
*bytes_written += bytes_used;
// cid
size_t cid_size = ipfs_cid_protobuf_encode_size(block->cid);
unsigned char cid[cid_size];
retVal = ipfs_cid_protobuf_encode(block->cid, cid, cid_size, &cid_size);
retVal = protobuf_encode_length_delimited(2, ipfs_block_message_fields[1], (char*)cid, cid_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
*bytes_written += bytes_used;
return 1;
}
/***
* Decode from a protobuf stream into a Block struct
* @param buffer the buffer to pull from
* @param buffer_length the length of the buffer
* @param block the block to fill
* @returns true(1) on success
*/
int ipfs_blocks_block_protobuf_decode(const unsigned char* buffer, const size_t buffer_length, struct Block** block) {
size_t pos = 0;
int retVal = 0;
unsigned char* temp_buffer = NULL;
size_t temp_size;
if (ipfs_blocks_block_new(block) == 0)
goto exit;
while(pos < buffer_length) {
size_t bytes_read = 0;
int field_no;
enum WireType field_type;
if (protobuf_decode_field_and_type(&buffer[pos], buffer_length, &field_no, &field_type, &bytes_read) == 0) {
goto exit;
}
pos += bytes_read;
switch(field_no) {
case (1): // data
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*block)->data), &((*block)->data_length), &bytes_read) == 0)
goto exit;
pos += bytes_read;
break;
case (2): // cid
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0)
goto exit;
pos += bytes_read;
if (ipfs_cid_protobuf_decode(temp_buffer, temp_size, &((*block)->cid)) == 0)
goto exit;
free(temp_buffer);
temp_buffer = NULL;
break;
}
}
retVal = 1;
exit:
if (retVal == 0) {
ipfs_blocks_block_free(*block);
}
if (temp_buffer != NULL)
free(temp_buffer);
return retVal;
}
/***
* Create a new block based on the incoming data
* @param data the data to base the block on
@ -16,35 +110,38 @@
* @param block a pointer to the struct Block that will be created
* @returns true(1) on success
*/
int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block) {
int ipfs_blocks_block_new(struct Block** block) {
// allocate memory for structure
(*block) = (struct Block*)malloc(sizeof(struct Block));
if ((*block) == NULL)
return 0;
(*block)->data = NULL;
(*block)->data_length = 0;
return 1;
}
int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, struct Block* block) {
// cid
unsigned char hash[32];
if (libp2p_crypto_hashing_sha256(data, data_size, &hash[0]) == 0) {
free(*block);
return 0;
}
if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &((*block)->cid)) == 0) {
free(*block);
if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &(block->cid)) == 0) {
return 0;
}
(*block)->data_length = data_size;
block->data_length = data_size;
(*block)->data = malloc(sizeof(unsigned char) * data_size);
if ( (*block)->data == NULL) {
ipfs_cid_free((*block)->cid);
free(*block);
block->data = malloc(sizeof(unsigned char) * data_size);
if ( block->data == NULL) {
ipfs_cid_free(block->cid);
return 0;
}
memcpy( (*block)->data, data, data_size);
memcpy( block->data, data, data_size);
return 1;
}

98
blocks/blockstore.c
View file

@ -6,6 +6,7 @@
#include "ipfs/blocks/block.h"
#include "ipfs/datastore/ds_helper.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
#include "ipfs/os/utils.h"
/**
* Delete a block based on its Cid
@ -25,14 +26,59 @@ int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo) {
return 0;
}
unsigned char* ipfs_blockstore_cid_to_base32(const struct Cid* cid) {
size_t key_length = libp2p_crypto_encoding_base32_encode_size(cid->hash_length);
unsigned char* buffer = (unsigned char*)malloc(key_length + 1);
if (buffer == NULL)
return NULL;
int retVal = ipfs_datastore_helper_ds_key_from_binary(cid->hash, cid->hash_length, &buffer[0], key_length, &key_length);
if (retVal == 0) {
free(buffer);
return NULL;
}
buffer[key_length] = 0;
return buffer;
}
char* ipfs_blockstore_path_get(const struct FSRepo* fs_repo, const char* filename) {
int filepath_size = strlen(fs_repo->path) + 12;
char filepath[filepath_size];
int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", filepath, filepath_size);
if (retVal == 0) {
free(filepath);
return 0;
}
int complete_filename_size = strlen(filepath) + strlen(filename) + 2;
char* complete_filename = (char*)malloc(complete_filename_size);
retVal = os_utils_filepath_join(filepath, filename, complete_filename, complete_filename_size);
return complete_filename;
}
/***
* Find a block based on its Cid
* @param cid the Cid to look for
* @param block where to put the data to be returned
* @returns true(1) on success
*/
int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo) {
return 0;
int ipfs_blockstore_get(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo) {
// get datastore key, which is a base32 key of the multihash
unsigned char* key = ipfs_blockstore_cid_to_base32(cid);
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
size_t file_size = os_utils_file_size(filename);
unsigned char buffer[file_size];
FILE* file = fopen(filename, "rb");
size_t bytes_read = fread(buffer, 1, file_size, file);
fclose(file);
int retVal = ipfs_blocks_block_protobuf_decode(buffer, bytes_read, block);
free(key);
free(filename);
return retVal;
}
/***
@ -42,14 +88,46 @@ int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_
*/
int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) {
// from blockstore.go line 118
// Get Datastore key, which is a base32 key of the binary,
size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length);
unsigned char key[key_length];
int retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length);
if (retVal == 0)
return 0;
int retVal = 0;
// send to Put with key
fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
// Get Datastore key, which is a base32 key of the multihash,
unsigned char* key = ipfs_blockstore_cid_to_base32(block->cid);
if (key == NULL) {
free(key);
return 0;
}
//TODO: put this in subdirectories
// turn the block into a binary array
size_t protobuf_len = ipfs_blocks_block_protobuf_encode_size(block);
unsigned char protobuf[protobuf_len];
retVal = ipfs_blocks_block_protobuf_encode(block, protobuf, protobuf_len, &protobuf_len);
if (retVal == 0) {
free(key);
return 0;
}
// now write byte array to file
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
if (filename == NULL) {
free(key);
return 0;
}
FILE* file = fopen(filename, "wb");
int bytes_written = fwrite(protobuf, 1, protobuf_len, file);
fclose(file);
if (bytes_written != protobuf_len) {
free(key);
free(filename);
return 0;
}
// send to Put with key (this is now done separately)
//fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
free(key);
free(filename);
return 1;
}

5
datastore/ds_helper.c
View file

@ -1,10 +1,11 @@
/**
* Some code to help with the datastore / blockstore interface
* NOTE: the datastore stores things under a multihash key
*/
#include "libp2p/crypto/encoding/base32.h"
#include "ipfs/datastore/ds_helper.h"
/**
* Generate a key based on the passed in binary_array
* Generate a base32 key based on the passed in binary_array (which is normally a multihash)
* @param binary_array what to base the key on
* @param array_length the size of the binary array
* @param results where the key will be put
@ -30,7 +31,7 @@ int ipfs_datastore_helper_ds_key_from_binary(unsigned char* binary_array, size_t
}
/**
* Generate a binary array based on the passed in datastore key
* Generate a binary array (normally a multihash) based on the passed in datastore key
* @param ds_key the base32 encoded key
* @param key_length the length of the base32 "string"
* @param binary_array where to put the decoded value

18
importer/importer.c
View file

@ -1,6 +1,7 @@
#include <stdio.h>
#include "ipfs/importer/importer.h"
#include "ipfs/merkledag/merkledag.h"
#define MAX_DATA_SIZE 262144 // 1024 * 256;
@ -14,26 +15,27 @@
* @param node the node to add to
* @returns number of bytes read
*/
size_t ipfs_import_chunk(FILE* file, struct Node* node) {
size_t ipfs_import_chunk(FILE* file, struct Node* node, struct FSRepo* fs_repo) {
unsigned char buffer[MAX_DATA_SIZE];
size_t bytes_read = fread(buffer, MAX_DATA_SIZE, 1, file);
size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file);
if (node->data_size == 0) {
ipfs_node_set_data(node, buffer, bytes_read);
if (bytes_read != MAX_DATA_SIZE) {
// persist
}
} else {
// create a new node, and link to the parent
struct Node* new_node = NULL;
ipfs_node_new_from_data(buffer, bytes_read, &new_node);
// persist
ipfs_merkledag_add(new_node, fs_repo);
// put link in node
struct NodeLink* new_link = NULL;
ipfs_node_link_new("", new_node->cached->hash, &new_link);
ipfs_node_add_link(node, new_link);
ipfs_node_free(new_node);
}
if (bytes_read != MAX_DATA_SIZE) {
// persist the main node
ipfs_merkledag_add(node, fs_repo);
}
return bytes_read;
}
@ -43,7 +45,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* node) {
* @param node the root node (could have links to others)
* @returns true(1) on success
*/
int ipfs_import_file(const char* fileName, struct Node** node) {
int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo) {
int retVal = 1;
int bytes_read = MAX_DATA_SIZE;
@ -54,7 +56,7 @@ int ipfs_import_file(const char* fileName, struct Node** node) {
// add all nodes
while ( bytes_read == MAX_DATA_SIZE) {
bytes_read = ipfs_import_chunk(file, *node);
bytes_read = ipfs_import_chunk(file, *node, fs_repo);
}
fclose(file);

31
include/ipfs/blocks/block.h
View file

@ -1,5 +1,6 @@
/***
* IPFS has the notion of storage blocks.
* Raw data with a multihash key (the Cid)
*/
#ifndef __IPFS_BLOCKS_BLOCK_H__
@ -20,7 +21,9 @@ struct Block {
* @param block a pointer to the struct Block that will be created
* @returns true(1) on success
*/
int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block);
int ipfs_blocks_block_new(struct Block** block);
int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, struct Block* block);
/***
* Free resources used by the creation of a block
@ -29,4 +32,30 @@ int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Bl
*/
int ipfs_blocks_block_free(struct Block* block);
/**
* Determine the approximate size of an encoded block
* @param block the block to measure
* @returns the approximate size needed to encode the protobuf
*/
size_t ipfs_blocks_block_protobuf_encode_size(const struct Block* block);
/**
* Encode the Block into protobuf format
* @param block the block to encode
* @param buffer the buffer to fill
* @param max_buffer_size the max size of the buffer
* @param bytes_written the number of bytes used
* @returns true(1) on success
*/
int ipfs_blocks_block_protobuf_encode(const struct Block* block, unsigned char* buffer, size_t max_buffer_size, size_t* bytes_written);
/***
* Decode from a protobuf stream into a Block struct
* @param buffer the buffer to pull from
* @param buffer_length the length of the buffer
* @param block the block to fill
* @returns true(1) on success
*/
int ipfs_blocks_block_protobuf_decode(const unsigned char* buffer, const size_t buffer_length, struct Block** block);
#endif

9
include/ipfs/blocks/blockstore.h
View file

@ -3,7 +3,10 @@
*/
#ifndef __IPFS_BLOCKS_BLOCKSTORE_H__
#ifndef __IPFS_BLOCKS_BLOCKSTORE_H__
#define __IPFS_BLOCKS_BLOCKSTORE_H__
#include "ipfs/cid/cid.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
/**
* Delete a block based on its Cid
@ -25,14 +28,14 @@ int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo);
* @param block where to put the data to be returned
* @returns true(1) on success
*/
int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo);
int ipfs_blockstore_get(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo);
/***
* Put a block in the blockstore
* @param block the block to store
* @returns true(1) on success
*/
int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo);
int ipfs_blockstore_put(struct Block* block, const struct FSRepo* fs_repo);
#endif

3
include/ipfs/importer/importer.h
View file

@ -2,6 +2,7 @@
#define __IPFS_IMPORTER_IMPORTER_H__
#include "ipfs/node/node.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
/**
* Creates a node based on an incoming file
@ -9,6 +10,6 @@
* @param node the root node (could have links to others)
* @returns true(1) on success
*/
int ipfs_import_file(const char* fileName, struct Node** node);
int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo);
#endif /* INCLUDE_IPFS_IMPORTER_IMPORTER_H_ */

9
include/ipfs/repo/fsrepo/fs_repo.h
View file

@ -64,4 +64,13 @@ int ipfs_repo_fsrepo_free(struct FSRepo* config);
*/
int ipfs_repo_fsrepo_init(struct FSRepo* config);
/***
* Write a block to the datastore and blockstore
* @param block the block to write
* @param fs_repo the repo to write to
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo);
int ipfs_repo_fsrepo_block_read(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo);
#endif /* fs_repo_h */

22
merkledag/merkledag.c
View file

@ -21,9 +21,11 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
// turn the node into a block
struct Block* block;
ipfs_blocks_block_new(protobuf, bytes_written, &block);
ipfs_blocks_block_new(&block);
ipfs_blocks_block_add_data(protobuf, bytes_written, block);
int retVal = fs_repo->config->datastore->datastore_put_block(block, fs_repo->config->datastore);
// write to block store & datastore
int retVal = ipfs_repo_fsrepo_block_write(block, fs_repo);
if (retVal == 0) {
ipfs_blocks_block_free(block);
return 0;
@ -45,17 +47,25 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
*/
int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo) {
int retVal = 1;
// look for the block
struct Block* block;
retVal = fs_repo->config->datastore->datastore_get_block(cid, &block, fs_repo->config->datastore);
size_t key_length = 100;
unsigned char key[key_length];
// look for the node in the datastore. If it is not there, it is not a node.
// If it exists, it is only a block.
retVal = fs_repo->config->datastore->datastore_get((char*)cid->hash, cid->hash_length, key, key_length, &key_length, fs_repo->config->datastore);
if (retVal == 0)
return 0;
// we have the block. Fill the node
// we have the record from the db. Go get the block from the blockstore
retVal = ipfs_repo_fsrepo_block_read(cid, &block, fs_repo);
// now convert the block into a node
ipfs_node_protobuf_decode(block->data, block->data_length, node);
// doesn't decode do this?
ipfs_node_set_cached(*node, cid);
// free resources
ipfs_blocks_block_free(block);
return retVal;

1
node/node.c
View file

@ -295,6 +295,7 @@ int ipfs_node_new(struct Node** node)
return 0;
(*node)->cached = NULL;
(*node)->data = NULL;
(*node)->data_size = 0;
(*node)->encoded = NULL;
(*node)->head_link = NULL;
return 1;

5
os/utils.c
View file

@ -62,11 +62,14 @@ int os_utils_directory_writeable(const char* path) {
}
int os_utils_file_size(const char* path) {
size_t file_size = 0;
// open file
FILE* in_file = fopen(path, "r");
if (in_file == NULL)
return 0;
// determine size
fseek(in_file, 0L, SEEK_END);
size_t file_size = ftell(in_file);
file_size = ftell(in_file);
fclose(in_file);
return file_size;
}

63
repo/fsrepo/fs_repo.c
View file

@ -1,6 +1,9 @@
#include <stdio.h>
#include <sys/stat.h>
#include "ipfs/blocks/blockstore.h"
#include "libp2p/crypto/encoding/base64.h"
#include "ipfs/datastore/ds_helper.h"
#include "ipfs/repo/config/datastore.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
#include "ipfs/os/utils.h"
@ -460,6 +463,18 @@ int ipfs_repo_fsrepo_datastore_init(struct FSRepo* fs_repo) {
return repo_fsrepo_lmdb_cast(fs_repo->config->datastore);
}
int ipfs_repo_fsrepo_blockstore_init(const struct FSRepo* fs_repo) {
size_t full_path_size = strlen(fs_repo->path) + 15;
char full_path[full_path_size];
int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", full_path, full_path_size);
if (retVal == 0)
return 0;
if (mkdir(full_path, S_IRWXU) != 0)
return 0;
return 1;
}
/**
* Initializes a new FSRepo at the given path with the provided config
* @param path the path to use
@ -477,11 +492,14 @@ int ipfs_repo_fsrepo_init(struct FSRepo* repo) {
if (retVal == 0)
return 0;
// TODO: Implement this method
retVal = ipfs_repo_fsrepo_datastore_init(repo);
if (retVal == 0)
return 0;
retVal = ipfs_repo_fsrepo_blockstore_init(repo);
if (retVal == 0)
return 0;
// write the version to a file for migrations (see repo/fsrepo/migrations/mfsr.go)
//TODO: mfsr.RepoPath(repo_path).WriteVersion(RepoVersion)
return 1;
@ -508,3 +526,46 @@ int fs_repo_write_config_file(char* path, struct RepoConfig* config) {
return retVal;
}
/***
* Write a block to the datastore and blockstore
* @param block the block to write
* @param fs_repo the repo to write to
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo) {
/**
* What is put in the blockstore is the block.
* What is put in the datastore is the multihash (the Cid) as the key,
* and the base32 encoded multihash as the value.
*/
int retVal = 1;
retVal = ipfs_blockstore_put(block, fs_repo);
if (retVal == 0)
return 0;
// take the cid, base32 it, and send both to the datastore
size_t fs_key_length = 100;
unsigned char fs_key[fs_key_length];
retVal = ipfs_datastore_helper_ds_key_from_binary(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, &fs_key_length);
if (retVal == 0)
return 0;
retVal = fs_repo->config->datastore->datastore_put(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, fs_repo->config->datastore);
if (retVal == 0)
return 0;
return 1;
}
int ipfs_repo_fsrepo_block_read(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo) {
int retVal = 0;
// get the base32 hash from the database
// We do this only to see if it is in the database
size_t fs_key_length = 100;
unsigned char fs_key[fs_key_length];
retVal = fs_repo->config->datastore->datastore_get((char*)cid->hash, cid->hash_length, fs_key, fs_key_length, &fs_key_length, fs_repo->config->datastore);
if (retVal == 0) // maybe it doesn't exist?
return 0;
// now get the block from the blockstore
retVal = ipfs_blockstore_get(cid, block, fs_repo);
return retVal;
}

5
repo/fsrepo/lmdb_datastore.c
View file

@ -1,5 +1,7 @@
/***
* Here are the wrappers for the lightning database
* NOTE: In this implementation, the database will contain the base32 encoded value
* of the multihash key if the file exists on disk.
*/
#include <stdlib.h>
@ -43,12 +45,13 @@ int repo_fsrepo_lmdb_get_block(const struct Cid* cid, struct Block** block, cons
}
// now copy the data
retVal = ipfs_blocks_block_new(db_value.mv_data, db_value.mv_size, block);
retVal = ipfs_blocks_block_new(block);
if (retVal == 0) {
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 0;
}
retVal = ipfs_blocks_block_add_data(db_value.mv_data, db_value.mv_size, *block);
// clean up
mdb_dbi_close(mdb_env, mdb_dbi);

77
test/node/test_importer.h Normal file
View file

@ -0,0 +1,77 @@
#include <stdio.h>
#include "ipfs/importer/importer.h"
#include "ipfs/merkledag/merkledag.h"
/***
* Helper to create a test file in the OS
*/
int create_file(const char* fileName, unsigned char* bytes, size_t num_bytes) {
FILE* file = fopen(fileName, "wb");
fwrite(bytes, num_bytes, 1, file);
fclose(file);
return 1;
}
int create_bytes(unsigned char* buffer, size_t num_bytes) {
int counter = 0;
for(int i = 0; i < num_bytes; i++) {
buffer[i] = counter++;
if (counter > 15)
counter = 0;
}
return 1;
}
int test_import_small_file() {
size_t bytes_size = 1000;
unsigned char file_bytes[bytes_size];
const char* fileName = "/tmp/test_import_small.tmp";
// create the necessary file
create_bytes(file_bytes, bytes_size);
create_file(fileName, file_bytes, bytes_size);
// get the repo
drop_and_build_repository("/tmp/.ipfs");
struct FSRepo* fs_repo;
ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, &fs_repo);
ipfs_repo_fsrepo_open(fs_repo);
// write to ipfs
struct Node* write_node;
if (ipfs_import_file(fileName, &write_node, fs_repo) == 0) {
ipfs_repo_fsrepo_free(fs_repo);
return 0;
}
// make sure all went okay
struct Node* read_node;
if (ipfs_merkledag_get(write_node->cached, &read_node, fs_repo) == 0) {
ipfs_repo_fsrepo_free(fs_repo);
ipfs_node_free(write_node);
return 0;
}
// compare data
if (write_node->data_size != bytes_size || write_node->data_size != read_node->data_size) {
printf("Data size of nodes are not equal or are incorrect. Should be %lu but are %lu\n", write_node->data_size, read_node->data_size);
ipfs_repo_fsrepo_free(fs_repo);
ipfs_node_free(write_node);
ipfs_node_free(read_node);
return 0;
}
for(int i = 0; i < bytes_size; i++) {
if (write_node->data[i] != read_node->data[i]) {
printf("Data within node is different at position %d\n", i);
ipfs_repo_fsrepo_free(fs_repo);
ipfs_node_free(write_node);
ipfs_node_free(read_node);
return 0;
}
}
return 1;
}

71
test/repo/test_repo_fsrepo.h
View file

@ -1,4 +1,5 @@
#include "ipfs/repo/fsrepo/fs_repo.h"
#include "../test_helper.h"
int test_repo_fsrepo_open_config() {
struct FSRepo* fs_repo = NULL;
@ -24,3 +25,73 @@ int test_repo_fsrepo_open_config() {
return 1;
}
int test_repo_fsrepo_write_read_block() {
struct Block* block = NULL;
struct FSRepo* fs_repo = NULL;
int retVal = 0;
// freshen the repository
retVal = drop_build_and_open_repo("/tmp/.ipfs", &fs_repo);
if (retVal == 0)
return 0;
// make some data
size_t data_size = 100;
unsigned char data[data_size];
int counter = 0;
for(int i = 0; i < data_size; i++) {
data[i] = counter++;
if (counter > 15)
counter = 0;
}
// create and write the block
retVal = ipfs_blocks_block_new(&block);
if (retVal == 0) {
ipfs_repo_fsrepo_free(fs_repo);
return 0;
}
retVal = ipfs_blocks_block_add_data(data, data_size, block);
if (retVal == 0) {
ipfs_repo_fsrepo_free(fs_repo);
return 0;
}
retVal = ipfs_repo_fsrepo_block_write(block, fs_repo);
if (retVal == 0) {
ipfs_repo_fsrepo_free(fs_repo);
ipfs_blocks_block_free(block);
return 0;
}
// retrieve the block
struct Block* results;
retVal = ipfs_repo_fsrepo_block_read(block->cid, &results, fs_repo);
if (retVal == 0) {
ipfs_repo_fsrepo_free(fs_repo);
ipfs_blocks_block_free(block);
return 0;
}
// compare the two blocks
retVal = 1;
if (block->data_length != results->data_length || block->data_length != data_size) {
printf("block data is of different length: %lu vs %lu\n", results->data_length, block->data_length);
retVal = 0;
}
for(size_t i = 0; i < block->data_length; i++) {
if (block->data[i] != results->data[i]) {
printf("Data is different at position %lu. Should be %02x but is %02x\n", i, block->data[i], results->data[i]);
retVal = 0;
break;
}
}
ipfs_repo_fsrepo_free(fs_repo);
ipfs_blocks_block_free(block);
ipfs_blocks_block_free(results);
return retVal;
}

32
test/storage/test_blocks.h
View file

@ -4,30 +4,48 @@ int test_blocks_new() {
const unsigned char* input = (const unsigned char*)"Hello, World!";
int retVal = 0;
struct Block* block;
retVal = ipfs_blocks_block_new(input, strlen((const char*)input) + 1, &block);
retVal = ipfs_blocks_block_new(&block);
if (retVal == 0)
return 0;
retVal = ipfs_blocks_block_add_data(input, strlen((const char*)input) + 1, block);
if (retVal == 0) {
ipfs_blocks_block_free(block);
return 0;
}
// now examine the block
if (strcmp((const char*)block->data, (const char*)input) != 0)
if (strcmp((const char*)block->data, (const char*)input) != 0) {
ipfs_blocks_block_free(block);
return 0;
}
if (block->data_length != strlen((const char*)input) + 1)
if (block->data_length != strlen((const char*)input) + 1) {
ipfs_blocks_block_free(block);
return 0;
}
if (block->cid->codec != CID_PROTOBUF)
if (block->cid->codec != CID_PROTOBUF) {
ipfs_blocks_block_free(block);
return 0;
}
if (block->cid->version != 0)
if (block->cid->version != 0) {
ipfs_blocks_block_free(block);
return 0;
}
if (block->cid->hash_length != 32)
if (block->cid->hash_length != 32) {
ipfs_blocks_block_free(block);
return 0;
}
unsigned char result_hash[32] = {33, 153, 66, 187, 124, 250, 87, 12, 12, 73, 43, 247, 175, 153, 10, 51, 192, 195, 218, 69, 220, 170, 105, 179, 195, 0, 203, 213, 172, 3, 244, 10 };
for(int i = 0; i < 32; i++) {
if (block->cid->hash[i] != result_hash[i])
if (block->cid->hash[i] != result_hash[i]) {
ipfs_blocks_block_free(block);
return 0;
}
}
retVal = ipfs_blocks_block_free(block);

25
test/storage/test_datastore.h
View file

@ -23,30 +23,43 @@ int test_ipfs_datastore_put() {
return 0;
// build the block
retVal = ipfs_blocks_block_new(input, strlen((char*)input), &block);
retVal = ipfs_blocks_block_new(&block);
if (retVal == 0)
return 0;
retVal = ipfs_blocks_block_add_data(input, strlen((char*)input), block);
if (retVal == 0) {
ipfs_blocks_block_free(block);
return 0;
}
// generate the key
size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length);
unsigned char key[key_length];
retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length);
if (retVal == 0)
if (retVal == 0) {
ipfs_blocks_block_free(block);
return 0;
}
// open the repository
struct FSRepo* fs_repo;
retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, &fs_repo);
if (retVal == 0)
if (retVal == 0) {
ipfs_blocks_block_free(block);
return 0;
}
retVal = ipfs_repo_fsrepo_open(fs_repo);
if (retVal == 0)
if (retVal == 0) {
ipfs_blocks_block_free(block);
return 0;
}
// send to Put with key
retVal = fs_repo->config->datastore->datastore_put((const unsigned char*)key, key_length, block->data, block->data_length, fs_repo->config->datastore);
if (retVal == 0)
if (retVal == 0) {
ipfs_blocks_block_free(block);
return 0;
}
// save the block

19
test/test_helper.c
View file

@ -122,3 +122,22 @@ int make_ipfs_repository(const char* path) {
int drop_and_build_repository(const char* path) {
return make_ipfs_repository(path);
}
int drop_build_and_open_repo(const char* path, struct FSRepo** fs_repo) {
int retVal = 0;
retVal = drop_and_build_repository("/tmp/.ipfs");
if (retVal == 0)
return 0;
retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, fs_repo);
if (retVal == 0)
return 0;
retVal = ipfs_repo_fsrepo_open(*fs_repo);
if (retVal == 0) {
free(*fs_repo);
return 0;
}
return 1;
}

6
test/test_helper.h
View file

@ -1,5 +1,11 @@
/**
* Helpers for testing
*/
/**
* Create a new repository in the directory, erasing old one
* NOTE: base directory must already exist
*/
int drop_and_build_repository(const char* dir);
int drop_build_and_open_repo(const char* path, struct FSRepo** fs_repo);

5
test/testit.c
View file

@ -3,6 +3,7 @@
#include "flatfs/test_flatfs.h"
#include "merkledag/test_merkledag.h"
#include "node/test_node.h"
#include "node/test_importer.h"
#include "repo/test_repo_bootstrap_peers.h"
#include "repo/test_repo_config.h"
#include "repo/test_repo_fsrepo.h"
@ -31,7 +32,9 @@ const char* names[] = {
"test_repo_config_write",
"test_repo_config_identity_new",
"test_repo_config_identity_private_key",
"test_repo_fsrepo_write_read_block",
"test_get_init_command",
"test_import_small_file",
"test_repo_fsrepo_open_config",
"test_flatfs_get_directory",
"test_flatfs_get_filename",
@ -58,7 +61,9 @@ int (*funcs[])(void) = {
test_repo_config_write,
test_repo_config_identity_new,
test_repo_config_identity_private_key,
test_repo_fsrepo_write_read_block,
test_get_init_command,
test_import_small_file,
test_repo_fsrepo_open_config,
test_flatfs_get_directory,
test_flatfs_get_filename,