agorise/c-ipfs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

More storage implementation

Browse source 
Successfully writing to lightningdb. Now to pull it back out. Also need
to write to the blockstore.
This commit is contained in:
John Jones 2016-11-30 11:46:41 -05:00
parent 4626b69381
commit b462d9ef53
29 changed files with 448 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
.cproject
View file

@ -14,7 +14,7 @@
</extensions> </extensions>
</storageModule> </storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0"> <storageModule moduleId="cdtBuildSystem" version="4.0.0">
<configuration buildProperties="" description="" id="cdt.managedbuild.toolchain.gnu.base.1186026282" name="Default" parent="org.eclipse.cdt.build.core.emptycfg"> <configuration artifactName="${ProjName}" buildProperties="" description="" id="cdt.managedbuild.toolchain.gnu.base.1186026282" name="Default" parent="org.eclipse.cdt.build.core.emptycfg">
<folderInfo id="cdt.managedbuild.toolchain.gnu.base.1186026282.828279619" name="/" resourcePath=""> <folderInfo id="cdt.managedbuild.toolchain.gnu.base.1186026282.828279619" name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.base.577806056" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.base"> <toolChain id="cdt.managedbuild.toolchain.gnu.base.577806056" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.base">
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.GNU_ELF" id="cdt.managedbuild.target.gnu.platform.base.309685154" name="Debug Platform" osList="linux,hpux,aix,qnx" superClass="cdt.managedbuild.target.gnu.platform.base"/> <targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.GNU_ELF" id="cdt.managedbuild.target.gnu.platform.base.309685154" name="Debug Platform" osList="linux,hpux,aix,qnx" superClass="cdt.managedbuild.target.gnu.platform.base"/>
@ -58,40 +58,6 @@
<resource resourceType="PROJECT" workspacePath="/c-ipfs"/> <resource resourceType="PROJECT" workspacePath="/c-ipfs"/>
</configuration> </configuration>
</storageModule> </storageModule>
<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets">
<buildTargets>
<target name="all" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
<buildCommand>make</buildCommand>
<buildTarget>all</buildTarget>
<stopOnError>true</stopOnError>
<useDefaultCommand>true</useDefaultCommand>
<runAllBuilders>true</runAllBuilders>
</target>
<target name="clean" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
<buildCommand>make</buildCommand>
<buildTarget>clean</buildTarget>
<stopOnError>true</stopOnError>
<useDefaultCommand>true</useDefaultCommand>
<runAllBuilders>true</runAllBuilders>
</target>
<target name="clean" path="thirdparty" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
<buildCommand>make</buildCommand>
<buildArguments/>
<buildTarget>clean</buildTarget>
<stopOnError>true</stopOnError>
<useDefaultCommand>false</useDefaultCommand>
<runAllBuilders>true</runAllBuilders>
</target>
<target name="clean" path="thirdparty/ipfsaddr" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
<buildCommand>make</buildCommand>
<buildArguments/>
<buildTarget>clean</buildTarget>
<stopOnError>true</stopOnError>
<useDefaultCommand>false</useDefaultCommand>
<runAllBuilders>true</runAllBuilders>
</target>
</buildTargets>
</storageModule>
<storageModule moduleId="scannerConfiguration"> <storageModule moduleId="scannerConfiguration">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/> <autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
<scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.base.1186026282;cdt.managedbuild.toolchain.gnu.base.1186026282.828279619;cdt.managedbuild.tool.gnu.cpp.compiler.base.1701788660;cdt.managedbuild.tool.gnu.cpp.compiler.input.1773271782"> <scannerConfigBuildInfo instanceId="cdt.managedbuild.toolchain.gnu.base.1186026282;cdt.managedbuild.toolchain.gnu.base.1186026282.828279619;cdt.managedbuild.tool.gnu.cpp.compiler.base.1701788660;cdt.managedbuild.tool.gnu.cpp.compiler.input.1773271782">
@ -101,4 +67,46 @@
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/> <autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
</scannerConfigBuildInfo> </scannerConfigBuildInfo>
</storageModule> </storageModule>
<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets">
<buildTargets>
<target name="all" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
<buildCommand>make</buildCommand>
<buildArguments/>
<buildTarget>all</buildTarget>
<stopOnError>true</stopOnError>
<useDefaultCommand>true</useDefaultCommand>
<runAllBuilders>true</runAllBuilders>
</target>
<target name="clean" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
<buildCommand>make</buildCommand>
<buildArguments/>
<buildTarget>clean</buildTarget>
<stopOnError>true</stopOnError>
<useDefaultCommand>true</useDefaultCommand>
<runAllBuilders>true</runAllBuilders>
</target>
<target name="rebuild" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
<buildCommand>make</buildCommand>
<buildArguments/>
<buildTarget>rebuild</buildTarget>
<stopOnError>true</stopOnError>
<useDefaultCommand>true</useDefaultCommand>
<runAllBuilders>true</runAllBuilders>
</target>
<target name="clean" path="thirdparty" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
<buildCommand>make</buildCommand>
<buildTarget>clean</buildTarget>
<stopOnError>true</stopOnError>
<useDefaultCommand>false</useDefaultCommand>
<runAllBuilders>true</runAllBuilders>
</target>
<target name="clean" path="thirdparty/ipfsaddr" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
<buildCommand>make</buildCommand>
<buildTarget>clean</buildTarget>
<stopOnError>true</stopOnError>
<useDefaultCommand>false</useDefaultCommand>
<runAllBuilders>true</runAllBuilders>
</target>
</buildTargets>
</storageModule>
</cproject> </cproject>

3
Makefile
View file

@ -3,6 +3,7 @@ DEBUG = true
export DEBUG export DEBUG
all: all:
cd blocks; make all;
cd cid; make all; cd cid; make all;
cd cmd; make all; cd cmd; make all;
cd commands; make all; cd commands; make all;
@ -16,6 +17,7 @@ all:
cd test; make all; cd test; make all;
clean: clean:
cd blocks; make clean;
cd cid; make clean; cd cid; make clean;
cd cmd; make clean; cd cmd; make clean;
cd commands; make clean; cd commands; make clean;
@ -28,3 +30,4 @@ clean:
cd thirdparty; make clean; cd thirdparty; make clean;
cd test; make clean; cd test; make clean;
rebuild: clean all

18
blocks/Makefile Normal file
View file

@ -0,0 +1,18 @@
CC = gcc
CFLAGS = -O0 -I../include -I../../c-libp2p/include -I../../c-multihash/include -I../../c-multiaddr/include
ifdef DEBUG
CFLAGS += -g3
endif
LFLAGS =
DEPS = ../include/blocks/block.h ../include/blocks/blockstore.h
OBJS = block.o blockstore.o
%.o: %.c $(DEPS)
$(CC) -c -o $@ $< $(CFLAGS)
all: $(OBJS)
clean:
rm -f *.o

19
blocks/block.c
View file

@ -3,6 +3,7 @@
*/ */
#include <stdlib.h> #include <stdlib.h>
#include <string.h>
#include "libp2p/crypto/sha256.h" #include "libp2p/crypto/sha256.h"
#include "ipfs/blocks/block.h" #include "ipfs/blocks/block.h"
@ -15,7 +16,7 @@
* @param block a pointer to the struct Block that will be created * @param block a pointer to the struct Block that will be created
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_blocks_block_new(unsigned char* data, size_t data_size, struct Block** block) { int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block) {
// allocate memory for structure // allocate memory for structure
(*block) = (struct Block*)malloc(sizeof(struct Block)); (*block) = (struct Block*)malloc(sizeof(struct Block));
@ -23,25 +24,27 @@ int ipfs_blocks_block_new(unsigned char* data, size_t data_size, struct Block**
return 0; return 0;
// cid // cid
char hash[32]; unsigned char hash[32];
if (libp2p_crypto_hashing_sha256(data, hash, 32) == 0) { if (libp2p_crypto_hashing_sha256(data, data_size, &hash[0]) == 0) {
free(*block); free(*block);
return 0; return 0;
} }
if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, (*block)->cid) == 0) { if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &((*block)->cid)) == 0) {
free(*block); free(*block);
return 0; return 0;
} }
block->data = malloc(sizeof(unsigned char) * data_size); (*block)->data_length = data_size;
if (block->data == NULL) {
ipfs_ci_free((*block)->cid); (*block)->data = malloc(sizeof(unsigned char) * data_size);
if ( (*block)->data == NULL) {
ipfs_cid_free((*block)->cid);
free(*block); free(*block);
return 0; return 0;
} }
memcpy(block->data, data, data_size); memcpy( (*block)->data, data, data_size);
return 1; return 1;
} }

25
blocks/blockstore.c
View file

@ -1,14 +1,18 @@
/*** /***
* a thin wrapper over a datastore for getting and putting block objects * a thin wrapper over a datastore for getting and putting block objects
*/ */
#include "libp2p/crypto/encoding/base32.h"
#include "ipfs/cid/cid.h"
#include "ipfs/blocks/block.h"
#include "ipfs/datastore/ds_helper.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
/** /**
* Delete a block based on its Cid * Delete a block based on its Cid
* @param cid the Cid to look for * @param cid the Cid to look for
* @param returns true(1) on success * @param returns true(1) on success
*/ */
int ipfs_blockstore_delete(struct Cid* cid) { int ipfs_blockstore_delete(struct Cid* cid, struct FSRepo* fs_repo) {
return 0; return 0;
} }
@ -17,7 +21,7 @@ int ipfs_blockstore_delete(struct Cid* cid) {
* @param cid the Cid to look for * @param cid the Cid to look for
* @returns true(1) if found * @returns true(1) if found
*/ */
int ipfs_blockstore_has(struct Cid* cid) { int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo) {
return 0; return 0;
} }
@ -27,7 +31,7 @@ int ipfs_blockstore_has(struct Cid* cid) {
* @param block where to put the data to be returned * @param block where to put the data to be returned
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_blockstore_get(struct Cid* cid, struct Block* block) { int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo) {
return 0; return 0;
} }
@ -36,9 +40,16 @@ int ipfs_blockstore_get(struct Cid* cid, struct Block* block) {
* @param block the block to store * @param block the block to store
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_blockstore_put(struct Block* block) { int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) {
// from blockstore.go line 118 // from blockstore.go line 118
// TODO: Get Datastore key // Get Datastore key, which is a base32 key of the binary,
// TODO: send to Put with key size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length);
unsigned char key[key_length];
int retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length);
if (retVal == 0)
return 0;
// send to Put with key
fs_repo->config->datastore->datastore_put(key, block, fs_repo->config->datastore);
return 0; return 0;
} }

7
cmd/ipfs/init.c
View file

@ -61,7 +61,7 @@ int do_init(FILE* out_file, char* repo_root, int empty, int num_bits_for_keypair
return 0; return 0;
//TODO: If the conf is null, make one //TODO: If the conf is null, make one
if ( conf->identity->peer_id == NULL) { if ( conf->identity->peer_id == NULL) {
int retVal = repo_config_init(conf, num_bits_for_keypair, repo_root); int retVal = ipfs_repo_config_init(conf, num_bits_for_keypair, repo_root);
if (retVal == 0) if (retVal == 0)
return 0; return 0;
} }
@ -87,11 +87,12 @@ int init_run(struct Request* request) {
// TODO: make sure offline // TODO: make sure offline
// TODO: check parameters for logic errors // TODO: check parameters for logic errors
// TODO: Initialize // TODO: Initialize
struct RepoConfig conf = {0}; struct RepoConfig* conf;
int retVal = ipfs_repo_config_new(&conf);
// TODO: handle files in request // TODO: handle files in request
// do the heavy lifting // do the heavy lifting
int num_bits_for_key_pair = request->cmd.options[0]->default_int_val; int num_bits_for_key_pair = request->cmd.options[0]->default_int_val;
return do_init(stdout, request->invoc_context->config_root, 1, num_bits_for_key_pair, &conf); return do_init(stdout, request->invoc_context->config_root, 1, num_bits_for_key_pair, conf);
} }
/*** /***

2
datastore/Makefile
View file

@ -1,5 +1,5 @@
CC = gcc CC = gcc
CFLAGS = -O0 -I../include -I../../c-libp2p/include -I../../c-multihash/include -I../../c-multiaddr/include CFLAGS = -O0 -I../include -I../../c-libp2p/include -I../../c-multihash/include -I../../c-multiaddr/include -I../../lmdb/libraries/liblmdb
ifdef DEBUG ifdef DEBUG
CFLAGS += -g3 CFLAGS += -g3

18
datastore/key.c Normal file
View file

@ -0,0 +1,18 @@
#include <string.h>
/**
* Constructs a new "clean" key. Will remove things like slashes
* @param input the input
* @param output the output
* @param max_output_length the amount of memory allocated for output
* @param actual_output_length the amount of bytes written to output
* @returns true(1) on success
*/
int ipfs_datastore_key_new(const char* input, char* output, size_t max_output_length, size_t* actual_output_length) {
//TODO: clean the input
if (strlen(input) + 1 > max_output_length)
return 0;
memcpy(output, input, strlen(input) + 1);
return 1;
}

5
include/ipfs/blocks/block.h
View file

@ -5,9 +5,12 @@
#ifndef __IPFS_BLOCKS_BLOCK_H__ #ifndef __IPFS_BLOCKS_BLOCK_H__
#define __IPFS_BLOCKS_BLOCK_H__ #define __IPFS_BLOCKS_BLOCK_H__
#include "ipfs/cid/cid.h"
struct Block { struct Block {
struct Cid* cid; struct Cid* cid;
unsigned char* data; unsigned char* data;
size_t data_length;
}; };
/*** /***
@ -17,7 +20,7 @@ struct Block {
* @param block a pointer to the struct Block that will be created * @param block a pointer to the struct Block that will be created
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_blocks_block_new(unsigned char* data, size_t data_size, struct Block** block); int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block);
/*** /***
* Free resources used by the creation of a block * Free resources used by the creation of a block

8
include/ipfs/blocks/blockstore.h
View file

@ -10,14 +10,14 @@
* @param cid the Cid to look for * @param cid the Cid to look for
* @param returns true(1) on success * @param returns true(1) on success
*/ */
int ipfs_blockstore_delete(struct Cid* cid); int ipfs_blockstore_delete(struct Cid* cid, struct FSRepo* fs_repo);
/*** /***
* Determine if the Cid can be found * Determine if the Cid can be found
* @param cid the Cid to look for * @param cid the Cid to look for
* @returns true(1) if found * @returns true(1) if found
*/ */
int ipfs_blockstore_has(struct Cid* cid); int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo);
/*** /***
* Find a block based on its Cid * Find a block based on its Cid
@ -25,14 +25,14 @@ int ipfs_blockstore_has(struct Cid* cid);
* @param block where to put the data to be returned * @param block where to put the data to be returned
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_blockstore_get(struct Cid* cid, struct Block* block); int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo);
/*** /***
* Put a block in the blockstore * Put a block in the blockstore
* @param block the block to store * @param block the block to store
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_blockstore_put(struct Block* block); int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo);
#endif #endif

2
include/ipfs/cid/cid.h
View file

@ -5,6 +5,8 @@
#ifndef __IPFS_CID_CID_H #ifndef __IPFS_CID_CID_H
#define __IPFS_CID_CID_H #define __IPFS_CID_CID_H
#include <stddef.h>
#define CID_PROTOBUF 0x70 #define CID_PROTOBUF 0x70
#define CID_CBOR 0x71 #define CID_CBOR 0x71
#define CID_RAW 0x72 #define CID_RAW 0x72

14
include/ipfs/datastore/key.h Normal file
View file

@ -0,0 +1,14 @@
#ifndef __IPFS_DATASTORE_KEY_H__
#define __IPFS_DATASTORE_KEY_H__
/**
* Constructs a new "clean" key. Will remove things like slashes
* @param input the input
* @param output the output
* @param max_output_length the amount of memory allocated for output
* @param actual_output_length the amount of bytes written to output
* @returns true(1) on success
*/
int ipfs_datastore_key_new(const char* input, char* output, size_t max_output_length, size_t* actual_output_length);
#endif

2
include/ipfs/repo/config/config.h
View file

@ -72,7 +72,7 @@ int config_path(char* config_root, char* extension, char* result, int max_len);
* @param num_bits_for_keypair number of bits for the key pair * @param num_bits_for_keypair number of bits for the key pair
* @returns true(1) on success, otherwise 0 * @returns true(1) on success, otherwise 0
*/ */
int repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path); int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path);
/*** /***
* Initialize memory for a RepoConfig struct * Initialize memory for a RepoConfig struct

3
include/ipfs/repo/config/datastore.h
View file

@ -2,6 +2,7 @@
#define __DATASTORE_H__ #define __DATASTORE_H__
#include <stdint.h> #include <stdint.h>
#include "ipfs/blocks/block.h"
//const char* datastore_default_directory = "datastore"; //const char* datastore_default_directory = "datastore";
@ -19,6 +20,8 @@ struct Datastore {
// function pointers for datastore operations // function pointers for datastore operations
int (*datastore_open)(int argc, char** argv, struct Datastore* datastore); int (*datastore_open)(int argc, char** argv, struct Datastore* datastore);
int (*datastore_close)(int argc, char** argv, struct Datastore* datastore); int (*datastore_close)(int argc, char** argv, struct Datastore* datastore);
int (*datastore_put)(const char* key, struct Block* block, struct Datastore* datastore);
//int (*datastore_get)(const char* key, struct Block* block);
// a handle to the datastore "context" used by the datastore // a handle to the datastore "context" used by the datastore
void* handle; void* handle;
}; };

14
include/ipfs/repo/fsrepo/fs_repo.h
View file

@ -16,13 +16,11 @@ struct FSRepo {
char* path; char* path;
struct IOCloser* lock_file; struct IOCloser* lock_file;
struct RepoConfig* config; struct RepoConfig* config;
struct Datastore* data_store;
}; };
/** /**
* opens a fsrepo * opens a fsrepo
* @param repo_path the path to the repo * @param repo the repo struct. Should contain the path. This method will do the rest
* @param repo where to store the repo info
* @return 0 if there was a problem, otherwise 1 * @return 0 if there was a problem, otherwise 1
*/ */
int ipfs_repo_fsrepo_open(struct FSRepo* repo); int ipfs_repo_fsrepo_open(struct FSRepo* repo);
@ -43,10 +41,12 @@ int fs_repo_is_initialized(char* repo_path);
int fs_repo_write_config_file(char* path, struct RepoConfig* config); int fs_repo_write_config_file(char* path, struct RepoConfig* config);
/** /**
* Initializes a new FSRepo at the given path with the provided config * constructs the FSRepo struct.
* @param repo_path the path to use * Remember: ipfs_repo_fsrepo_free must be called
* @param config the information for the config file * @param repo_path the path to the repo
* @returns true(1) on success * @param config the optional config file. NOTE: if passed, fsrepo_free will free resources of the RepoConfig.
* @param repo the struct to allocate memory for
* @returns false(0) if something bad happened, otherwise true(1)
*/ */
int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** fs_repo); int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** fs_repo);

7
include/ipfs/repo/fsrepo/lmdb_datastore.h
View file

@ -27,4 +27,11 @@ int repo_fsrepro_lmdb_open(int argc, char** argv, struct Datastore* datastore);
*/ */
int repo_fsrepo_lmdb_close(int argc, char** argv, struct Datastore* datastore); int repo_fsrepo_lmdb_close(int argc, char** argv, struct Datastore* datastore);
/***
* Creates the directory
* @param datastore contains the path that needs to be created
* @returns true(1) on success
*/
int repo_fsrepo_lmdb_create_directory(struct Datastore* datastore);
#endif #endif

26
include/ipfs/repo/repo.h
View file

@ -1,26 +0,0 @@
#ifndef __REPO_H__
#define __REPO_H__
#include <stdint.h>
#include "config/config.h"
/**
* Get the config
* @param config a place to put the buffer (must have been pre-allocated)
* @returns 0 on error
*/
int repo_get_config(struct RepoConfig* config);
/**
* Retrieves the config
* @param config a place to get the information
* @returns 0 on error
*/
int repo_set_config(struct RepoConfig* config);
int repo_set_config_key(char* key, void* value);
int repo_get_config_key(char* key, void* value);
int repo_get_datastore(struct Datastore* datastore);
int repo_get_storage_usage(uint64_t* usage);
#endif // __REPO_H__

2
repo/Makefile
View file

@ -6,7 +6,7 @@ CFLAGS += -g3
endif endif
DEPS = DEPS =
OBJS = repo.o OBJS =
%.o: %.c $(DEPS) %.o: %.c $(DEPS)
$(CC) -c -o $@ $< $(CFLAGS) $(CC) -c -o $@ $< $(CFLAGS)

2
repo/config/config.c
View file

@ -87,7 +87,7 @@ int repo_config_get_file_name(char* path, char** result) {
* @param num_bits_for_keypair number of bits for the key pair * @param num_bits_for_keypair number of bits for the key pair
* @returns true(1) on success, otherwise 0 * @returns true(1) on success, otherwise 0
*/ */
int repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path) { int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path) {
// identity // identity
int retVal = repo_config_identity_init(config->identity, num_bits_for_keypair); int retVal = repo_config_identity_init(config->identity, num_bits_for_keypair);
if (retVal == 0) if (retVal == 0)

1
repo/config/datastore.c
View file

@ -40,6 +40,7 @@ int ipfs_repo_config_datastore_new(struct Datastore** datastore) {
if (*datastore == NULL) if (*datastore == NULL)
return 0; return 0;
(*datastore)->path = NULL; (*datastore)->path = NULL;
(*datastore)->handle = NULL;
return 1; return 1;
} }

30
repo/fsrepo/fs_repo.c
View file

@ -109,7 +109,7 @@ int repo_config_write_config_file(char* full_filename, struct RepoConfig* config
* Remember: ipfs_repo_fsrepo_free must be called * Remember: ipfs_repo_fsrepo_free must be called
* @param repo_path the path to the repo * @param repo_path the path to the repo
* @param config the optional config file. NOTE: if passed, fsrepo_free will free resources of the RepoConfig. * @param config the optional config file. NOTE: if passed, fsrepo_free will free resources of the RepoConfig.
* @param repo the struct to fill in * @param repo the struct to allocate memory for
* @returns false(0) if something bad happened, otherwise true(1) * @returns false(0) if something bad happened, otherwise true(1)
*/ */
int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** repo) { int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** repo) {
@ -140,11 +140,6 @@ int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRe
return 0; return 0;
} }
} }
if (ipfs_repo_config_datastore_new(&((*repo)->data_store)) == 0) {
free(repo_path);
ipfs_repo_config_free((*repo)->config);
return 0;
}
return 1; return 1;
} }
@ -159,8 +154,6 @@ int ipfs_repo_fsrepo_free(struct FSRepo* repo) {
free(repo->path); free(repo->path);
if (repo->config != NULL) if (repo->config != NULL)
ipfs_repo_config_free(repo->config); ipfs_repo_config_free(repo->config);
if (repo->data_store != NULL)
ipfs_repo_config_datastore_free(repo->data_store);
free(repo); free(repo);
} }
return 1; return 1;
@ -347,7 +340,7 @@ int fs_repo_open_config(struct FSRepo* repo) {
return 0; return 0;
} }
// fill FSRepo struct // fill FSRepo struct
repo->config = malloc(sizeof(struct RepoConfig)); // allocation done by fsrepo_new... repo->config = malloc(sizeof(struct RepoConfig));
// Identity // Identity
int curr_pos = _find_token(data, tokens, num_tokens, 0, "Identity"); int curr_pos = _find_token(data, tokens, num_tokens, 0, "Identity");
if (curr_pos < 0) { if (curr_pos < 0) {
@ -389,7 +382,7 @@ int fs_repo_open_config(struct FSRepo* repo) {
* @returns true(1) on success * @returns true(1) on success
*/ */
int fs_repo_setup_lmdb_datastore(struct FSRepo* repo) { int fs_repo_setup_lmdb_datastore(struct FSRepo* repo) {
return repo_fsrepo_lmdb_cast(repo->data_store); return repo_fsrepo_lmdb_cast(repo->config->datastore);
} }
/*** /***
@ -401,10 +394,7 @@ int fs_repo_open_datastore(struct FSRepo* repo) {
int argc = 0; int argc = 0;
char** argv = NULL; char** argv = NULL;
// copy struct from config area to this area if (strncmp(repo->config->datastore->type, "lmdb", 4) == 0) {
repo->data_store = repo->config->datastore;
if (strncmp(repo->data_store->type, "lmdb", 4) == 0) {
// this is a LightningDB. Open it. // this is a LightningDB. Open it.
int retVal = fs_repo_setup_lmdb_datastore(repo); int retVal = fs_repo_setup_lmdb_datastore(repo);
if (retVal == 0) if (retVal == 0)
@ -414,7 +404,7 @@ int fs_repo_open_datastore(struct FSRepo* repo) {
return 0; return 0;
} }
int retVal = repo->data_store->datastore_open(argc, argv, repo->data_store); int retVal = repo->config->datastore->datastore_open(argc, argv, repo->config->datastore);
// do specific datastore cleanup here if needed // do specific datastore cleanup here if needed
@ -463,6 +453,14 @@ int fs_repo_is_initialized(char* repo_path) {
return fs_repo_is_initialized_unsynced(repo_path); return fs_repo_is_initialized_unsynced(repo_path);
} }
int ipfs_repo_fsrepo_datastore_init(struct FSRepo* fs_repo) {
// make the directory
repo_fsrepo_lmdb_create_directory(fs_repo->config->datastore);
// fill in the function prototypes
repo_fsrepo_lmdb_cast(fs_repo->config->datastore);
}
/** /**
* Initializes a new FSRepo at the given path with the provided config * Initializes a new FSRepo at the given path with the provided config
* @param path the path to use * @param path the path to use
@ -481,7 +479,7 @@ int ipfs_repo_fsrepo_init(struct FSRepo* repo) {
return 0; return 0;
// TODO: Implement this method // TODO: Implement this method
//retVal = fs_repo_defaultds_init(path, config); retVal = ipfs_repo_fsrepo_datastore_init(repo);
if (retVal == 0) if (retVal == 0)
return 0; return 0;

58
repo/fsrepo/lmdb_datastore.c
View file

@ -3,10 +3,55 @@
*/ */
#include <stdlib.h> #include <stdlib.h>
#include <sys/stat.h>
#include <errno.h>
#include <string.h>
#include "lmdb.h" #include "lmdb.h"
#include "ipfs/repo/fsrepo/lmdb_datastore.h" #include "ipfs/repo/fsrepo/lmdb_datastore.h"
/**
* Write a block to the datastore with the specified key
* @param key the key
* @param block the block to be written
* @returns true(1) on success
*/
int repo_fsrepo_lmdb_put(const char* key, struct Block* block, struct Datastore* datastore) {
int retVal;
MDB_txn* mdb_txn;
MDB_dbi mdb_dbi;
struct MDB_val db_key;
struct MDB_val db_value;
MDB_env* mdb_env = (MDB_env*)datastore->handle;
if (mdb_env == NULL)
return 0;
// open transaction
retVal = mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn);
if (retVal != 0)
return 0;
retVal = mdb_dbi_open(mdb_txn, NULL, 0, &mdb_dbi);
if (retVal != 0)
return 0;
// write
db_key.mv_size = strlen(key) + 1;
db_key.mv_data = (char*)key;
db_value.mv_size = block->data_length;
db_value.mv_data = block->data;
retVal = mdb_put(mdb_txn, mdb_dbi, &db_key, &db_value, MDB_NODUPDATA);
if (retVal != 0)
return 0;
// cleanup
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 1;
}
/** /**
* Open an lmdb database with the given parameters. * Open an lmdb database with the given parameters.
* Note: for now, the parameters are not used * Note: for now, the parameters are not used
@ -23,7 +68,7 @@ int repo_fsrepro_lmdb_open(int argc, char** argv, struct Datastore* datastore) {
} }
// open the environment // open the environment
retVal = mdb_env_open(mdb_env, datastore->path, 0, 755); retVal = mdb_env_open(mdb_env, datastore->path, 0, S_IRWXU);
if (retVal < 0) { if (retVal < 0) {
mdb_env_close(mdb_env); mdb_env_close(mdb_env);
return 0; return 0;
@ -55,6 +100,17 @@ int repo_fsrepo_lmdb_close(int argc, char** argv, struct Datastore* datastore) {
int repo_fsrepo_lmdb_cast(struct Datastore* datastore) { int repo_fsrepo_lmdb_cast(struct Datastore* datastore) {
datastore->datastore_open = &repo_fsrepro_lmdb_open; datastore->datastore_open = &repo_fsrepro_lmdb_open;
datastore->datastore_close = &repo_fsrepo_lmdb_close; datastore->datastore_close = &repo_fsrepo_lmdb_close;
datastore->datastore_put = &repo_fsrepo_lmdb_put;
//datastore->datastore_get = &repo_fsrepo_lmdb_get;
return 1; return 1;
} }
/***
* Creates the directory
* @param datastore contains the path that needs to be created
* @returns true(1) on success
*/
int repo_fsrepo_lmdb_create_directory(struct Datastore* datastore) {
return mkdir(datastore->path, S_IRWXU) == 0;
}

6
repo/repo.c
View file

@ -1,6 +0,0 @@
#include "ipfs/repo/repo.h"
int repo_get_config(struct RepoConfig* config) {
return 0;
}

3
test/Makefile
View file

@ -8,7 +8,8 @@ OBJS = testit.o ../cmd/ipfs/init.o ../commands/argument.o ../commands/command_op
../repo/config/bootstrap_peers.o ../repo/config/datastore.o ../repo/config/gateway.o \ ../repo/config/bootstrap_peers.o ../repo/config/datastore.o ../repo/config/gateway.o \
../repo/config/addresses.o ../repo/config/swarm.o ../repo/config/peer.o \ ../repo/config/addresses.o ../repo/config/swarm.o ../repo/config/peer.o \
../thirdparty/ipfsaddr/ipfs_addr.o ../cid/cid.o ../multibase/multibase.o \ ../thirdparty/ipfsaddr/ipfs_addr.o ../cid/cid.o ../multibase/multibase.o \
../flatfs/flatfs.o ../datastore/ds_helper.o ../flatfs/flatfs.o ../blocks/block.o ../blocks/blockstore.o \
../datastore/ds_helper.o
%.o: %.c $(DEPS) %.o: %.c $(DEPS)
$(CC) -c -o $@ $< $(CFLAGS) $(CC) -c -o $@ $< $(CFLAGS)

4
test/repo/test_repo_config.h
View file

@ -32,7 +32,7 @@ int test_repo_config_init() {
if (retVal == 0) if (retVal == 0)
return 0; return 0;
retVal = repo_config_init(repoConfig, 2048, "/Users/JohnJones/.ipfs"); retVal = ipfs_repo_config_init(repoConfig, 2048, "/Users/JohnJones/.ipfs");
if (retVal == 0) if (retVal == 0)
return 0; return 0;
@ -77,7 +77,7 @@ int test_repo_config_write() {
// now build a new one // now build a new one
struct RepoConfig* repoConfig; struct RepoConfig* repoConfig;
ipfs_repo_config_new(&repoConfig); ipfs_repo_config_new(&repoConfig);
if (!repo_config_init(repoConfig, 2048, "/tmp/.ipfs")) { if (!ipfs_repo_config_init(repoConfig, 2048, "/tmp/.ipfs")) {
ipfs_repo_config_free(repoConfig); ipfs_repo_config_free(repoConfig);
return 0; return 0;
} }

36
test/storage/test_blocks.h Normal file
View file

@ -0,0 +1,36 @@
#include "ipfs/blocks/block.h"
int test_blocks_new() {
const char* input = "Hello, World!";
int retVal = 0;
struct Block* block;
retVal = ipfs_blocks_block_new(input, strlen(input) + 1, &block);
if (retVal == 0)
return 0;
// now examine the block
if (strcmp(block->data, input) != 0)
return 0;
if (block->data_length != strlen(input) + 1)
return 0;
if (block->cid->codec != CID_PROTOBUF)
return 0;
if (block->cid->version != 0)
return 0;
if (block->cid->hash_length != 32)
return 0;
unsigned char result_hash[32] = {33, 153, 66, 187, 124, 250, 87, 12, 12, 73, 43, 247, 175, 153, 10, 51, 192, 195, 218, 69, 220, 170, 105, 179, 195, 0, 203, 213, 172, 3, 244, 10 };
for(int i = 0; i < 32; i++) {
if (block->cid->hash[i] != result_hash[i])
return 0;
}
retVal = ipfs_blocks_block_free(block);
return 1;
}

20
test/storage/test_blockstore.h Normal file
View file

@ -0,0 +1,20 @@
#include <string.h>
#include "ipfs/blocks/blockstore.h"
#include "ipfs/blocks/block.h"
int test_blockstore_put() {
const unsigned char* input_string = "Hello, World!";
struct Block* input_block;
int retVal = 0;
retVal = ipfs_blocks_block_new(input_string, strlen((const char*)input_string) + 1, &input_block);
if (retVal == 0)
return 0;
retVal = ipfs_blockstore_put(input_block);
if (retVal == 0)
return 0;
return 1;
}

148
test/storage/test_datastore.h Normal file
View file

@ -0,0 +1,148 @@
#include "libp2p/crypto/encoding/base32.h"
#include "ipfs/datastore/ds_helper.h"
#include "ipfs/blocks/block.h"
#include "ipfs/repo/config/config.h"
#include "ipfs/repo/fsrepo/fs_repo.h"
#include <dirent.h>
#include <sys/stat.h>
int remove_directory(const char *path)
{
DIR *d = opendir(path);
size_t path_len = strlen(path);
int r = -1;
if (d)
{
struct dirent *p;
r = 0;
while (!r && (p=readdir(d)))
{
int r2 = -1;
char *buf;
size_t len;
/* Skip the names "." and ".." as we don't want to recurse on them. */
if (!strcmp(p->d_name, ".") || !strcmp(p->d_name, ".."))
{
continue;
}
len = path_len + strlen(p->d_name) + 2;
buf = malloc(len);
if (buf)
{
struct stat statbuf;
snprintf(buf, len, "%s/%s", path, p->d_name);
if (!stat(buf, &statbuf))
{
if (S_ISDIR(statbuf.st_mode))
{
r2 = remove_directory(buf);
}
else
{
r2 = unlink(buf);
}
}
free(buf);
}
r = r2;
}
closedir(d);
}
if (!r)
{
r = rmdir(path);
}
return r;
}
int make_ipfs_repository(struct FSRepo* fs_repo) {
int retVal;
struct RepoConfig* repo_config;
unlink("/tmp/.ipfs/config");
remove_directory("/tmp/.ipfs/datastore");
remove_directory("/tmp/.ipfs/blockstore");
// build a default repo config
retVal = ipfs_repo_config_new(&repo_config);
if (retVal == 0)
return 0;
retVal = ipfs_repo_config_init(repo_config, 2048, "/tmp/.ipfs");
if (retVal == 0)
return 0;
// now the fs_repo
retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", repo_config, &fs_repo);
if (retVal == 0)
return 0;
// this builds a new repo
retVal = ipfs_repo_fsrepo_init(fs_repo);
if (retVal == 0)
return 0;
// clean up
ipfs_repo_fsrepo_free(fs_repo);
// this is cleaned up by fsrepo_free
//ipfs_repo_config_free(repo_config);
// make sure the repository exists
retVal = os_utils_file_exists("/tmp/.ipfs/config");
return retVal;
}
int test_ipfs_datastore_put() {
struct Block* block;
int retVal;
const unsigned char* input = "Hello, world!";
// build the ipfs repository, then shut it down, so we can start fresh
struct FSRepo* fs_repo;
retVal = make_ipfs_repository(fs_repo);
ipfs_repo_fsrepo_free(fs_repo);
if (retVal == 0)
return 0;
// build the block
retVal = ipfs_blocks_block_new(input, strlen((char*)input), &block);
if (retVal == 0)
return 0;
// generate the key
size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length);
unsigned char key[key_length];
retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length);
if (retVal == 0)
return 0;
// open the repository
retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, &fs_repo);
if (retVal == 0)
return 0;
retVal = ipfs_repo_fsrepo_open(fs_repo);
if (retVal == 0)
return 0;
// send to Put with key
retVal = fs_repo->config->datastore->datastore_put(key, block, fs_repo->config->datastore);
if (retVal == 0)
return 0;
// save the block
// check the results
return 1;
}

6
test/testit.c
View file

@ -6,6 +6,8 @@
#include "cid/test_cid.h" #include "cid/test_cid.h"
#include "flatfs/test_flatfs.h" #include "flatfs/test_flatfs.h"
#include "storage/test_ds_helper.h" #include "storage/test_ds_helper.h"
#include "storage/test_datastore.h"
#include "storage/test_blocks.h"
int testit(const char* name, int (*func)(void)) { int testit(const char* name, int (*func)(void)) {
printf("Testing %s...\n", name); printf("Testing %s...\n", name);
@ -19,6 +21,7 @@ int testit(const char* name, int (*func)(void)) {
int main(int argc, char** argv) { int main(int argc, char** argv) {
int counter = 0; int counter = 0;
/*
counter += testit("test_cid_new_free", test_cid_new_free); counter += testit("test_cid_new_free", test_cid_new_free);
counter += testit("test_cid_cast_multihash", test_cid_cast_multihash); counter += testit("test_cid_cast_multihash", test_cid_cast_multihash);
counter += testit("test_cid_cast_non_multihash", test_cid_cast_non_multihash); counter += testit("test_cid_cast_non_multihash", test_cid_cast_non_multihash);
@ -35,6 +38,9 @@ int main(int argc, char** argv) {
counter += testit("test_flatfs_get_filename", test_flatfs_get_filename); counter += testit("test_flatfs_get_filename", test_flatfs_get_filename);
counter += testit("test_flatfs_get_full_filename", test_flatfs_get_full_filename); counter += testit("test_flatfs_get_full_filename", test_flatfs_get_full_filename);
counter += testit("test_ds_key_from_binary", test_ds_key_from_binary); counter += testit("test_ds_key_from_binary", test_ds_key_from_binary);
counter += testit("test_blocks_new", test_blocks_new);
*/
counter += testit("test_ipfs_datastore_put", test_ipfs_datastore_put);
if (counter > 0) { if (counter > 0) {
printf("***** There were %d failed test(s) *****\n", counter); printf("***** There were %d failed test(s) *****\n", counter);
} else { } else {