Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Key motivations inspired by the broader Lance roadmap<sup>[1](https://github.com
## Features

- Unified schema for agent messages (`ContextRecord`) with optional embeddings and metadata.
- GraphRAG-friendly `relationships` column for directed edges such as
`{"target_id": "...", "relation": "cites", "weight": 0.75}`.
- Automatic versioning via Lance manifests with `checkout(version)` support.
- Background compaction to optimize storage and read performance.
- Remote persistence on any `object_store` backend (S3, GCS, Azure Blob, ...)
Expand Down Expand Up @@ -103,6 +105,14 @@ ctx.add(
embedding=runbook_embedding,
bot_id="support-bot",
session_id="incident-123",
relationships=[
{
"target_id": "docs://runbooks/service-a",
"relation": "cites",
"weight": 0.92,
},
{"target_id": "service://service-a", "relation": "describes"},
],
metadata={
"tenant": "example-org",
"scope": "team",
Expand All @@ -123,7 +133,9 @@ hits = ctx.search(
runbook_embedding,
limit=10,
filters={"tenant": "example-org", "content_type": "text/plain"},
include_relationships=True,
)
service_context = ctx.related("service://service-a", relation="describes")

from PIL import Image
image = Image.new("RGB", (2, 2), color="teal")
Expand All @@ -138,6 +150,9 @@ ctx.add_many([
"content": "Chunk 1 from a runbook",
"content_type": "text/markdown",
"session_id": "runbook-import",
"relationships": [
{"target_id": "service://service-a", "relation": "describes"}
],
},
{
"role": "source",
Expand Down Expand Up @@ -223,7 +238,7 @@ physical cleanup policies remove them.
### Rust

```rust
use lance_context::{ContextStore, ContextRecord, StateMetadata};
use lance_context::{ContextStore, ContextRecord, Relationship, StateMetadata};
use chrono::Utc;

# tokio_test::block_on(async {
Expand All @@ -241,6 +256,11 @@ let record = ContextRecord {
custom: None,
}),
metadata: None,
relationships: vec![Relationship {
target_id: "service://service-a".into(),
relation: "mentions".into(),
weight: None,
}],
expires_at: None,
retention_policy: None,
lifecycle_status: "active".into(),
Expand Down
15 changes: 15 additions & 0 deletions crates/lance-context-api/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ pub trait ContextStoreApi {
&self,
query: &[f32],
limit: Option<usize>,
include_relationships: bool,
) -> impl Future<Output = ContextResult<Vec<SearchResultDto>>> + Send;

fn version(&self) -> u64;
Expand Down Expand Up @@ -103,6 +104,14 @@ pub struct StateMetadataDto {
pub custom: Option<String>,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RelationshipDto {
pub target_id: String,
pub relation: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub weight: Option<f32>,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AddRecordRequest {
#[serde(default = "default_role")]
Expand Down Expand Up @@ -130,6 +139,8 @@ pub struct AddRecordRequest {
pub state_metadata: Option<StateMetadataDto>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Value>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub relationships: Vec<RelationshipDto>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expires_at: Option<DateTime<Utc>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
Expand Down Expand Up @@ -178,6 +189,8 @@ pub struct RecordDto {
pub state_metadata: Option<StateMetadataDto>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Value>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub relationships: Vec<RelationshipDto>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expires_at: Option<DateTime<Utc>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
Expand Down Expand Up @@ -216,6 +229,8 @@ pub struct SearchRequest {
pub query: Vec<f32>,
#[serde(default = "default_search_limit")]
pub limit: usize,
#[serde(default)]
pub include_relationships: bool,
}

#[derive(Debug, Serialize, Deserialize)]
Expand Down
2 changes: 2 additions & 0 deletions crates/lance-context-client/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,12 @@ impl ContextStoreApi for RemoteContextStore {
&self,
query: &[f32],
limit: Option<usize>,
include_relationships: bool,
) -> ContextResult<Vec<SearchResultDto>> {
let req = SearchRequest {
query: query.to_vec(),
limit: limit.unwrap_or(10),
include_relationships,
};
let resp = self
.client
Expand Down
44 changes: 39 additions & 5 deletions crates/lance-context-core/src/api_impl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ use uuid::Uuid;

use lance_context_api::{
AddRecordRequest, AddRecordsResponse, CompactRequest, CompactResponse, CompactStatsResponse,
ContextError, ContextResult, ContextStoreApi, RecordDto, SearchResultDto, StateMetadataDto,
ContextError, ContextResult, ContextStoreApi, RecordDto, RelationshipDto, SearchResultDto,
StateMetadataDto,
};

use crate::record::{ContextRecord, StateMetadata, LIFECYCLE_ACTIVE};
use crate::record::{ContextRecord, Relationship, StateMetadata, LIFECYCLE_ACTIVE};
use crate::store::{CompactionConfig, ContextStore};

impl ContextStoreApi for ContextStore {
Expand All @@ -33,6 +34,12 @@ impl ContextStoreApi for ContextStore {
custom: sm.custom.clone(),
}),
metadata: r.metadata.clone(),
relationships: r
.relationships
.iter()
.cloned()
.map(dto_to_relationship)
.collect(),
expires_at: r.expires_at,
retention_policy: r.retention_policy.clone(),
lifecycle_status: LIFECYCLE_ACTIVE.to_string(),
Expand Down Expand Up @@ -76,15 +83,21 @@ impl ContextStoreApi for ContextStore {
&self,
query: &[f32],
limit: Option<usize>,
include_relationships: bool,
) -> ContextResult<Vec<SearchResultDto>> {
let results = ContextStore::search(self, query, limit)
.await
.map_err(to_ctx_err)?;
Ok(results
.into_iter()
.map(|sr| SearchResultDto {
record: record_to_dto(sr.record),
distance: sr.distance,
.map(|mut sr| {
if !include_relationships {
sr.record.relationships.clear();
}
SearchResultDto {
record: record_to_dto(sr.record),
distance: sr.distance,
}
})
.collect())
}
Expand Down Expand Up @@ -136,6 +149,22 @@ impl ContextStoreApi for ContextStore {
}
}

fn dto_to_relationship(r: RelationshipDto) -> Relationship {
Relationship {
target_id: r.target_id,
relation: r.relation,
weight: r.weight,
}
}

fn relationship_to_dto(r: Relationship) -> RelationshipDto {
RelationshipDto {
target_id: r.target_id,
relation: r.relation,
weight: r.weight,
}
}

fn record_to_dto(r: ContextRecord) -> RecordDto {
RecordDto {
id: r.id,
Expand All @@ -156,6 +185,11 @@ fn record_to_dto(r: ContextRecord) -> RecordDto {
custom: sm.custom,
}),
metadata: r.metadata,
relationships: r
.relationships
.into_iter()
.map(relationship_to_dto)
.collect(),
expires_at: r.expires_at,
retention_policy: r.retention_policy,
lifecycle_status: r.lifecycle_status,
Expand Down
4 changes: 2 additions & 2 deletions crates/lance-context-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ mod store;

pub use context::{Context, ContextEntry, Snapshot};
pub use record::{
ContextRecord, LifecycleQueryOptions, MetadataFilter, RecordFilters, SearchResult,
StateMetadata, LIFECYCLE_ACTIVE, LIFECYCLE_CONTRADICTED,
ContextRecord, LifecycleQueryOptions, MetadataFilter, RecordFilters, Relationship,
SearchResult, StateMetadata, LIFECYCLE_ACTIVE, LIFECYCLE_CONTRADICTED,
};
pub use store::{
CompactionConfig, CompactionStats, ContextStore, ContextStoreOptions, IdIndexType,
Expand Down
12 changes: 12 additions & 0 deletions crates/lance-context-core/src/record.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;

Expand All @@ -16,6 +17,15 @@ pub struct StateMetadata {
pub custom: Option<String>,
}

/// Directed relationship from this record to another graph node.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Relationship {
pub target_id: String,
pub relation: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub weight: Option<f32>,
}

/// User-facing representation of a context entry written to storage.
#[derive(Debug, Clone)]
pub struct ContextRecord {
Expand All @@ -28,6 +38,7 @@ pub struct ContextRecord {
pub role: String,
pub state_metadata: Option<StateMetadata>,
pub metadata: Option<Value>,
pub relationships: Vec<Relationship>,
pub expires_at: Option<DateTime<Utc>>,
pub retention_policy: Option<String>,
pub lifecycle_status: String,
Expand Down Expand Up @@ -236,6 +247,7 @@ mod tests {
"tags": ["runbook", "ownership"],
"confidence": 0.92
})),
relationships: Vec::new(),
expires_at: None,
retention_policy: None,
lifecycle_status: LIFECYCLE_ACTIVE.to_string(),
Expand Down
Loading
Loading