liz/veilid
1
0
Fork 0
Code Pull Requests Packages Releases Activity

debugging

Browse Source
This commit is contained in:
John Smith
2023-05-21 22:16:27 +01:00
parent f31044e8a3
commit f54a6fcf31
19 changed files with 139 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
veilid-core/src/core_context.rs
+6
View File
@@ -70,6 +70,7 @@ impl ServicesContext {
trace!("init protected store");
let protected_store = ProtectedStore::new(self.config.clone());
if let Err(e) = protected_store.init().await {
error!("failed to init protected store: {}", e);
self.shutdown().await;
return Err(e);
}
@@ -79,6 +80,7 @@ impl ServicesContext {
trace!("init table store");
let table_store = TableStore::new(self.config.clone());
if let Err(e) = table_store.init().await {
error!("failed to init table store: {}", e);
self.shutdown().await;
return Err(e);
}
@@ -92,6 +94,7 @@ impl ServicesContext {
protected_store.clone(),
);
if let Err(e) = crypto.init().await {
error!("failed to init crypto: {}", e);
self.shutdown().await;
return Err(e);
}
@@ -101,6 +104,7 @@ impl ServicesContext {
trace!("init block store");
let block_store = BlockStore::new(self.config.clone());
if let Err(e) = block_store.init().await {
error!("failed to init block store: {}", e);
self.shutdown().await;
return Err(e);
}
@@ -116,6 +120,7 @@ impl ServicesContext {
self.block_store.clone().unwrap(),
);
if let Err(e) = storage_manager.init().await {
error!("failed to init storage manager: {}", e);
self.shutdown().await;
return Err(e);
}
@@ -133,6 +138,7 @@ impl ServicesContext {
crypto,
);
if let Err(e) = attachment_manager.init(update_callback).await {
error!("failed to init attachment manager: {}", e);
self.shutdown().await;
return Err(e);
}
veilid-core/src/intf/native/table_store.rs
+7 -4
View File
@@ -45,10 +45,13 @@ impl TableStore {
}
pub(crate) async fn terminate(&self) {
assert!(
self.inner.lock().opened.is_empty(),
"all open databases should have been closed"
);
let inner = self.inner.lock();
if !inner.opened.is_empty() {
panic!(
"all open databases should have been closed: {:?}",
inner.opened
);
}
}
pub(crate) fn on_table_db_drop(&self, table: String) {
veilid-core/src/intf/table_db.rs
+31 -31
View File
@@ -55,13 +55,13 @@ impl TableDB {
}
/// Get the total number of columns in the TableDB
pub fn get_column_count(&self) -> EyreResult<u32> {
pub fn get_column_count(&self) -> VeilidAPIResult<u32> {
let db = &self.unlocked_inner.database;
db.num_columns().wrap_err("failed to get column count: {}")
db.num_columns().map_err(VeilidAPIError::from)
}
/// Get the list of keys in a column of the TableDB
pub async fn get_keys(&self, col: u32) -> EyreResult<Vec<Box<[u8]>>> {
pub async fn get_keys(&self, col: u32) -> VeilidAPIResult<Vec<Box<[u8]>>> {
let db = self.unlocked_inner.database.clone();
let mut out: Vec<Box<[u8]>> = Vec::new();
db.iter(col, None, |kv| {
@@ -69,7 +69,7 @@ impl TableDB {
Ok(Option::<()>::None)
})
.await
.wrap_err("failed to get keys for column")?;
.map_err(VeilidAPIError::from)?;
Ok(out)
}
@@ -80,15 +80,15 @@ impl TableDB {
}
/// Store a key with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store(&self, col: u32, key: &[u8], value: &[u8]) -> EyreResult<()> {
pub async fn store(&self, col: u32, key: &[u8], value: &[u8]) -> VeilidAPIResult<()> {
let db = self.unlocked_inner.database.clone();
let mut dbt = db.transaction();
dbt.put(col, key, value);
db.write(dbt).await.wrap_err("failed to store key")
db.write(dbt).await.map_err(VeilidAPIError::generic)
}
/// Store a key in rkyv format with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
pub async fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
@@ -97,30 +97,30 @@ impl TableDB {
let db = self.unlocked_inner.database.clone();
let mut dbt = db.transaction();
dbt.put(col, key, v.as_slice());
db.write(dbt).await.wrap_err("failed to store key")
db.write(dbt).await.map_err(VeilidAPIError::generic)
}
/// Store a key in json format with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
pub async fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: serde::Serialize,
{
let v = serde_json::to_vec(value)?;
let v = serde_json::to_vec(value).map_err(VeilidAPIError::internal)?;
let db = self.unlocked_inner.database.clone();
let mut dbt = db.transaction();
dbt.put(col, key, v.as_slice());
db.write(dbt).await.wrap_err("failed to store key")
db.write(dbt).await.map_err(VeilidAPIError::generic)
}
/// Read a key from a column in the TableDB immediately.
pub async fn load(&self, col: u32, key: &[u8]) -> EyreResult<Option<Vec<u8>>> {
pub async fn load(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<Vec<u8>>> {
let db = self.unlocked_inner.database.clone();
db.get(col, key).await.wrap_err("failed to get key")
db.get(col, key).await.map_err(VeilidAPIError::from)
}
/// Read an rkyv key from a column in the TableDB immediately
pub async fn load_rkyv<T>(&self, col: u32, key: &[u8]) -> EyreResult<Option<T>>
pub async fn load_rkyv<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
@@ -128,33 +128,33 @@ impl TableDB {
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let out = match self.load(col, key).await? {
Some(v) => from_rkyv(v)?,
Some(v) => Some(from_rkyv(v)?),
None => None,
};
Ok(out)
}
/// Read an serde-json key from a column in the TableDB immediately
pub async fn load_json<T>(&self, col: u32, key: &[u8]) -> EyreResult<Option<T>>
pub async fn load_json<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
T: for<'de> serde::Deserialize<'de>,
{
let out = match self.load(col, key).await? {
Some(v) => serde_json::from_slice(&v)?,
Some(v) => Some(serde_json::from_slice(&v).map_err(VeilidAPIError::internal)?),
None => None,
};
Ok(out)
}
/// Delete key with from a column in the TableDB
pub async fn delete(&self, col: u32, key: &[u8]) -> EyreResult<Option<Vec<u8>>> {
pub async fn delete(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<Vec<u8>>> {
let db = self.unlocked_inner.database.clone();
let old_value = db.delete(col, key).await.wrap_err("failed to delete key")?;
let old_value = db.delete(col, key).await.map_err(VeilidAPIError::from)?;
Ok(old_value)
}
/// Delete rkyv key with from a column in the TableDB
pub async fn delete_rkyv<T>(&self, col: u32, key: &[u8]) -> EyreResult<Option<T>>
pub async fn delete_rkyv<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
@@ -162,21 +162,21 @@ impl TableDB {
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let db = self.unlocked_inner.database.clone();
let old_value = match db.delete(col, key).await.wrap_err("failed to delete key")? {
Some(v) => from_rkyv(v)?,
let old_value = match db.delete(col, key).await.map_err(VeilidAPIError::from)? {
Some(v) => Some(from_rkyv(v)?),
None => None,
};
Ok(old_value)
}
/// Delete serde-json key with from a column in the TableDB
pub async fn delete_json<T>(&self, col: u32, key: &[u8]) -> EyreResult<Option<T>>
pub async fn delete_json<T>(&self, col: u32, key: &[u8]) -> VeilidAPIResult<Option<T>>
where
T: for<'de> serde::Deserialize<'de>,
{
let db = self.unlocked_inner.database.clone();
let old_value = match db.delete(col, key).await.wrap_err("failed to delete key")? {
Some(v) => serde_json::from_slice(&v)?,
let old_value = match db.delete(col, key).await.map_err(VeilidAPIError::from)? {
Some(v) => Some(serde_json::from_slice(&v).map_err(VeilidAPIError::internal)?),
None => None,
};
Ok(old_value)
@@ -219,18 +219,18 @@ impl TableDBTransaction {
}
/// Commit the transaction. Performs all actions atomically.
pub async fn commit(self) -> EyreResult<()> {
pub async fn commit(self) -> VeilidAPIResult<()> {
let dbt = {
let mut inner = self.inner.lock();
inner
.dbt
.take()
.ok_or_else(|| eyre!("transaction already completed"))?
.ok_or_else(|| VeilidAPIError::generic("transaction already completed"))?
};
let db = self.db.unlocked_inner.database.clone();
db.write(dbt)
.await
.wrap_err("commit failed, transaction lost")
.map_err(|e| VeilidAPIError::generic(format!("commit failed, transaction lost: {}", e)))
}
/// Rollback the transaction. Does nothing to the TableDB.
@@ -246,7 +246,7 @@ impl TableDBTransaction {
}
/// Store a key in rkyv format with a value in a column in the TableDB
pub fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
pub fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
@@ -257,11 +257,11 @@ impl TableDBTransaction {
}
/// Store a key in rkyv format with a value in a column in the TableDB
pub fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
pub fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> VeilidAPIResult<()>
where
T: serde::Serialize,
{
let v = serde_json::to_vec(value)?;
let v = serde_json::to_vec(value).map_err(VeilidAPIError::internal)?;
let mut inner = self.inner.lock();
inner.dbt.as_mut().unwrap().put(col, key, v.as_slice());
Ok(())
veilid-core/src/intf/wasm/table_store.rs
+3 -3
View File
@@ -1,10 +1,10 @@
use super::*;
use crate::intf::table_db::TableDBInner;
use crate::intf::table_db::TableDBUnlockedInner;
pub use crate::intf::table_db::{TableDB, TableDBTransaction};
use keyvaluedb_web::*;
struct TableStoreInner {
opened: BTreeMap<String, Weak<Mutex<TableDBInner>>>,
opened: BTreeMap<String, Weak<TableDBUnlockedInner>>,
}
#[derive(Clone)]
@@ -95,7 +95,7 @@ impl TableStore {
};
}
}
let db = Database::open(table_name.clone(), column_count)
let db = Database::open(table_name.clone(), column_count, false)
.await
.wrap_err("failed to open tabledb")?;
trace!(
veilid-core/src/routing_table/bucket_entry.rs
+1 -1
View File
@@ -538,7 +538,7 @@ impl BucketEntryInner {
}
}
pub fn set_our_node_info_ts(&mut self, routing_domain: RoutingDomain, seen_ts: Timestamp) {
pub fn set_seen_our_node_info_ts(&mut self, routing_domain: RoutingDomain, seen_ts: Timestamp) {
match routing_domain {
RoutingDomain::LocalNetwork => {
self.local_network.last_seen_our_node_info_ts = seen_ts;
veilid-core/src/routing_table/debug.rs
+4 -1
View File
@@ -142,7 +142,10 @@ impl RoutingTable {
.latency
.as_ref()
.map(|l| {
format!("{:.2}ms", timestamp_to_secs(l.average.as_u64()))
format!(
"{:.2}ms",
timestamp_to_secs(l.average.as_u64()) * 1000.0
)
})
.unwrap_or_else(|| "???.??ms".to_string())
})
veilid-core/src/routing_table/mod.rs
+2 -2
View File
@@ -324,11 +324,11 @@ impl RoutingTable {
let dbx = tdb.transact();
if let Err(e) = dbx.store_rkyv(0, b"serialized_bucket_map", &serialized_bucket_map) {
dbx.rollback();
return Err(e);
return Err(e.into());
}
if let Err(e) = dbx.store_rkyv(0, b"all_entry_bytes", &all_entry_bytes) {
dbx.rollback();
return Err(e);
return Err(e.into());
}
dbx.commit().await?;
Ok(())
veilid-core/src/routing_table/node_ref.rs
+2 -2
View File
@@ -170,8 +170,8 @@ pub trait NodeRefBase: Sized {
) -> bool {
self.operate(|_rti, e| e.has_seen_our_node_info_ts(routing_domain, our_node_info_ts))
}
fn set_our_node_info_ts(&self, routing_domain: RoutingDomain, seen_ts: Timestamp) {
self.operate_mut(|_rti, e| e.set_our_node_info_ts(routing_domain, seen_ts));
fn set_seen_our_node_info_ts(&self, routing_domain: RoutingDomain, seen_ts: Timestamp) {
self.operate_mut(|_rti, e| e.set_seen_our_node_info_ts(routing_domain, seen_ts));
}
fn network_class(&self, routing_domain: RoutingDomain) -> Option<NetworkClass> {
self.operate(|_rt, e| e.node_info(routing_domain).map(|n| n.network_class()))
veilid-core/src/routing_table/routing_table_inner.rs
+7
View File
@@ -557,11 +557,18 @@ impl RoutingTableInner {
.map(|nr| nr.same_bucket_entry(&entry))
.unwrap_or(false);
if e.needs_ping(cur_ts, is_our_relay) {
debug!("needs_ping: {}", e.best_node_id());
return true;
}
// If we need a ping because this node hasn't seen our latest node info, then do it
if let Some(own_node_info_ts) = own_node_info_ts {
if !e.has_seen_our_node_info_ts(routing_domain, own_node_info_ts) {
//xxx remove this when we fix
debug!(
"!has_seen_our_node_info_ts: {} own_node_info_ts={}",
e.best_node_id(),
own_node_info_ts
);
return true;
}
}
veilid-core/src/rpc_processor/mod.rs
+2 -1
View File
@@ -1350,7 +1350,8 @@ impl RPCProcessor {
// Update the 'seen our node info' timestamp to determine if this node needs a
// 'node info update' ping
if let Some(sender_nr) = &opt_sender_nr {
sender_nr.set_our_node_info_ts(routing_domain, operation.target_node_info_ts());
sender_nr
.set_seen_our_node_info_ts(routing_domain, operation.target_node_info_ts());
}
// Make the RPC message
veilid-core/src/storage_manager/storage_manager_inner.rs
+9 -1
View File
@@ -154,7 +154,15 @@ impl StorageManagerInner {
async fn load_metadata(&mut self) -> EyreResult<()> {
if let Some(metadata_db) = &self.metadata_db {
self.offline_subkey_writes = metadata_db.load_rkyv(0, b"offline_subkey_writes").await?.unwrap_or_default();
self.offline_subkey_writes = match metadata_db.load_rkyv(0, b"offline_subkey_writes").await {
Ok(v) => v.unwrap_or_default(),
Err(_) => {
if let Err(e) = metadata_db.delete(0,b"offline_subkey_writes").await {
debug!("offline_subkey_writes format changed, clearing: {}", e);
}
Default::default()
}
}
}
Ok(())
}
veilid-core/src/tests/common/test_table_store.rs
+1 -4
View File
@@ -110,10 +110,7 @@ pub async fn test_store_delete_load(ts: TableStore) {
);
assert_eq!(db.load(2, b"baz").await.unwrap(), Some(b"QWERTY".to_vec()));
assert_eq!(
db.delete(1, b"bar").await.unwrap(),
Some(b"QWERTYUIOP".to_vec())
);
assert_eq!(db.delete(1, b"bar").await.unwrap(), Some(b"FNORD".to_vec()));
assert_eq!(db.delete(1, b"bar").await.unwrap(), None);
assert!(
db.delete(4, b"bar").await.is_err(),
veilid-core/src/veilid_api/error.rs
+33
View File
@@ -221,3 +221,36 @@ impl VeilidAPIError {
}
}
}
pub type VeilidAPIResult<T> = Result<T, VeilidAPIError>;
impl From<std::io::Error> for VeilidAPIError {
fn from(e: std::io::Error) -> Self {
match e.kind() {
std::io::ErrorKind::TimedOut => VeilidAPIError::timeout(),
std::io::ErrorKind::ConnectionRefused => VeilidAPIError::no_connection(e.to_string()),
std::io::ErrorKind::ConnectionReset => VeilidAPIError::no_connection(e.to_string()),
#[cfg(feature = "io_error_more")]
std::io::ErrorKind::HostUnreachable => VeilidAPIError::no_connection(e.to_string()),
#[cfg(feature = "io_error_more")]
std::io::ErrorKind::NetworkUnreachable => VeilidAPIError::no_connection(e.to_string()),
std::io::ErrorKind::ConnectionAborted => VeilidAPIError::no_connection(e.to_string()),
std::io::ErrorKind::NotConnected => VeilidAPIError::no_connection(e.to_string()),
std::io::ErrorKind::AddrInUse => VeilidAPIError::no_connection(e.to_string()),
std::io::ErrorKind::AddrNotAvailable => VeilidAPIError::no_connection(e.to_string()),
#[cfg(feature = "io_error_more")]
std::io::ErrorKind::NetworkDown => VeilidAPIError::no_connection(e.to_string()),
#[cfg(feature = "io_error_more")]
std::io::ErrorKind::ReadOnlyFilesystem => VeilidAPIError::internal(e.to_string()),
#[cfg(feature = "io_error_more")]
std::io::ErrorKind::NotSeekable => VeilidAPIError::internal(e.to_string()),
#[cfg(feature = "io_error_more")]
std::io::ErrorKind::FilesystemQuotaExceeded => VeilidAPIError::internal(e.to_string()),
#[cfg(feature = "io_error_more")]
std::io::ErrorKind::Deadlock => VeilidAPIError::internal(e.to_string()),
std::io::ErrorKind::Unsupported => VeilidAPIError::internal(e.to_string()),
std::io::ErrorKind::OutOfMemory => VeilidAPIError::internal(e.to_string()),
_ => VeilidAPIError::generic(e.to_string()),
}
}
}
veilid-core/src/veilid_api/serialize_helpers/veilid_rkyv.rs
+5 -5
View File
@@ -122,14 +122,14 @@ impl<E: Debug + fmt::Display> std::error::Error for VeilidRkyvError<E> {}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
pub fn to_rkyv<T>(value: &T) -> EyreResult<Vec<u8>>
pub fn to_rkyv<T>(value: &T) -> VeilidAPIResult<Vec<u8>>
where
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let mut serializer = DefaultVeilidRkyvSerializer::default();
serializer
.serialize_value(value)
.wrap_err("failed to serialize object")?;
.map_err(|e| VeilidAPIError::generic(format!("failed to serialize object: {}", e)))?;
Ok(serializer
.into_inner()
.into_serializer()
@@ -137,7 +137,7 @@ where
.to_vec())
}
pub fn from_rkyv<T>(bytes: Vec<u8>) -> EyreResult<T>
pub fn from_rkyv<T>(bytes: Vec<u8>) -> VeilidAPIResult<T>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
@@ -145,7 +145,7 @@ where
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
rkyv::check_archived_root::<T>(&bytes)
.map_err(|e| eyre!("checkbytes failed: {}", e))?
.map_err(|e| VeilidAPIError::generic(format!("checkbytes failed: {}", e)))?
.deserialize(&mut VeilidSharedDeserializeMap::default())
.map_err(|e| eyre!("failed to deserialize: {}", e))
.map_err(|e| VeilidAPIError::generic(format!("failed to deserialize: {}", e)))
}