---
title: 'Brain OS Onbaording'
disqus: hackmd
---

Brain OS Onboarding
===

## Table of Contents

[TOC]

## Business Onboardin Steps

### 1. Knowledge Onboarding

1. Define entity schema
2. Source one or multiple excel/csv for data
3. Configure excel/csv columns mapping
4. Upload file using Brain UI workbench

### 2. Trasaction Onboarding

1. Configure Azure bucket or kafka topics for ingestion process
2. Define column or field mapping
3. Define ingestion event schema

### 3. Feature definition and DAG

1. Define target feature that needed to be generated
2. Using brain os building blocks create a pseudocode defining data transformation from ingestion to target features in form of DAG
3. Define input and output event schema for each processes in DAG
4. Define configuration of each process in brain ui workbench

### 4. Function definition

1. Identify machine learning, deep learning, rules and optimization functions for a domain
2. Configure these function using brain ui workbench with its input and output features

Knowledge Onboarding
---
#### Step 1 :
Identify all entities for a domain.
#### Step 2 :
Define all **quantities** (schemas) and **units** needed to create above entities
:::info
Current focus is on **Atomic, Spatial and Temporal** Quantity
:::

message BrainQuantitySchema {
  jio.brain.proto.base.BrainToken token = 1;
  jio.brain.proto.base.BrainQuantityType quantity_type = 3;
  oneof quantity_schema_is_one_of {
    BrainAtomicQuantitySchema	atomic = 4;
    BrainCompoundQuantitySchema	compound = 5;
    BrainTemporalQuantitySchema	temporal = 6;
    BrainSpatialQuantitySchema spatial = 7;
    BrainMediaQuantitySchema media = 8;
  }
}
message BrainAtomicQuantitySchema {
  BrainAtomicQuantityType atomic_type = 1;
  oneof quantity_schema_is_one_of {
    BrainNumericQuantitySchema numeric_schema = 2;
    BrainSymbolicQuantitySchema symbolic_schema = 3;
    BrainOrdinalQuantitySchema ordinal_schema = 4;
    BrainBinnedQuantitySchema binned_schema = 5;
    BrainBinaryQuantitySchema binary_schema = 6;
  }
}
message BrainTemporalQuantitySchema {
  oneof temporal_is_one_of {
    BrainTemporalInstanceQuantitySchema instance = 1;
    BrainTemporalRangeQuantitySchema range = 2;
    BrainTemporalDurationQuantitySchema duration = 3;
    BrainTemporalContextQuantitySchema context = 4;
  }
}
message BrainSpatialQuantitySchema {
  oneof location_schema_is_one_of {
    bool lat_long = 1;
    BrainSpatialGeohashQuantitySchema geohash = 2;
    BrainSpatialHierarchyLevel location_hierarchy_level = 3;
  }
}

#### Step 3 :
Define entity schema for all entities. It include biz id's, name, attributes and predicates

message BrainEntitySchema {
  jio.brain.proto.base.BrainToken token = 1;
  jio.brain.proto.base.BrainToken primary_biz_id = 2;
  repeated jio.brain.proto.base.BrainToken biz_type = 4;
  BrainAttributeSetSchema attribute = 5;
  BrainPredicateSetSchema predicate = 6;
  repeated BrainAttributeFamilySchema attribute_family_set = 7;
}
message BrainAttributeSetSchema {
  map<string, BrainAttributeSchema> attribute = 5;
}
message BrainPredicateSetSchema {
  map<string, BrainPredicateSchema> predicate = 5;
}
message BrainAttributeSchema {
  jio.brain.proto.base.BrainToken attribute_token = 1; 
  optional jio.brain.proto.base.BrainToken unit_token = 2;
  oneof attribute_is_one_of {
    jio.brain.proto.quantity.BrainQuantitySchema absolute = 3;
    BrainQualifiedAttributeSchema qualifiers = 4;
  }
  bool is_knowledge_attribute = 5;
}
message BrainQualifiedAttributeSchema {
  jio.brain.proto.quantity.BrainQuantitySchema value = 1;
  map<string, BrainAttributeQualifierSchema> qualifiers = 2;
}
message BrainAttributeQualifierSchema {
  jio.brain.proto.base.BrainToken token = 1; 
  jio.brain.proto.quantity.BrainQuantitySchema quantity = 2;
  optional jio.brain.proto.base.BrainToken unit = 3;
}
message BrainPredicateSchema {
  jio.brain.proto.base.BrainToken predicate = 1;
  jio.brain.proto.base.BrainToken object = 2;
  map<string, BrainPredicateQualifierSchema> qualifiers = 3;
}
message BrainPredicateQualifierSchema {
  jio.brain.proto.base.BrainToken token = 1; 
  jio.brain.proto.quantity.BrainQuantitySchema quantity = 2;
  optional jio.brain.proto.base.BrainToken unit = 3;
}

#### Step 4 :
Source knowledge data in excel/csv form. There can be one or multiple files and each file contain only 1 kind of entity.

:::warning
Only **csv/excel** data import is supported
:::

#### Step 5 :
Define column mapping of above excel file. Each column represent either biz id or name or an attribute.

#### Step 6 :
There will be seperate file for each kind of predicate. Predicate ingestion file will have only two mandatory columns where first column represent the **subject biz_id** and second column represent **object biz_id**. All subsequent columns represent predicate qualifiers.

#### Step 7 :
:::warning
Ingestion of **attribute** qualifier not supported for this release.
:::
Define column mapping of predicate qualifiers for above excel file similar to attribute mapping defined above.

#### Step 8 :
:::danger
Entity ingestion should happen prior to predicate ingestion.
:::
Upload excel files using brain workbench. This will add entity data to knowledge repository.

Transaction Onboarding
---
#### Step 1 :
Identify data source. It can be either file pushed on azure storage at every predefined interval of time or data coming on some kafka topic.
:::warning
Only **kafka** and **csv file** on azure blob storage is supported
:::

message BrainProcessSchema {
  jio.brain.proto.base.BrainToken token = 1;
  string input_topic = 2;
  jio.brain.proto.event.BrainEventSchema input_event_schema = 3;
  map<string, jio.brain.proto.event.BrainEventSchema> output_event_schema = 4; 
  jio.brain.proto.base.BrainProcessType brain_process_type = 5;
  oneof config {
    BrainEnrichProcessSchema enrich = 6;
    BrainFilterProcessSchema filter = 7;
    BrainPivotProcessSchema pivot = 8;
    BrainReduceProcessSchema reduce = 9;
    BrainIngestProcessSchema ingest = 10;
    BrainQuantizeProcessSchema quantize = 11;
    BrainComputeProcessSchema compute = 12;
    BrainFunctionProcessSchema function = 13;
    BrainRepositoryProcessSchema repository = 14;
  }
}
message BrainIngestProcessSchema {
  oneof source {
    KafkaTransport kafka_transport = 2;
    FileTransport file_transport = 3;
    JsonFileTransport json_transport = 4;
    BlobStorageTransport blob_transport = 5;
  }
}
message KafkaTransport {
  string broker_host = 1;
  uint32 broker_port = 2;
  repeated FieldMapping mappings = 3;
}
message BlobStorageTransport {
  string bucket_name = 1;
  string prefix = 2;
  string suffix = 3;
  string secret_key = 4;
  string access_key = 5;
  repeated FieldMapping mappings = 6;
}

#### Step 2 :

Each field in csv file or in kafka event map to either biz id of entity, attribute of entity or property of transaction event.

message FieldMapping {
  uint32 field_index = 1; // column index in case of csv or excel
  string field_name = 2; // key name in case of kafka
  string column_unit = 3; // unit of this column
  jio.brain.proto.base.BrainInputDataType input_data_type = 4; // data type of input - will always be string in case of csv
  string format = 5; // date format or number format in case of date or number
  FieldType field_type = 6;
  bool is_optional = 7;
  oneof element {
    BizIdField entity_biz_id = 8;
    AttributeField entity_attribute = 9;
    AttributeQualifierField entity_attribute_qualifier = 10;
    PropertyField property = 11;
  }
}

#### Step 3 :

Define transaction event schema which is result of ingestion process.

message BrainEventSchema {
  jio.brain.proto.base.BrainToken token = 1;
  oneof event_schema_is_one_of {
    BrainIngestionEventSchema ingestion_event = 2;
    BrainTransactionEventSchema transaction_event = 3;
    BrainIotSensorEventSchema iot_event = 4;
    BrainEventKeyValueSchema key_value_event = 5; // same as pivot event
    BrainEventFeatureKeyValueSchema feature_event = 6;
    BrainTextEventSchema text_event = 7;
    BrainAudioEventSchema audio_event = 8;
    BrainImageEventSchema image_event = 9;
    BrainVideoEventSchema video_event = 10;
    BrainClassificationEventSchema classification_event = 11;
    BrainRegressionEventSchema regression_event = 12;
    BrainOutlierEventSchema outlier_event = 13;
    BrainRecommendationEventSchema recommendation_event = 14;
    BrainDecisionEventSchema decision_event = 15;
    BrainMetricEventSchema metric_event = 16;
  }
}
message BrainTransactionEventSchema {
  BrainEntityStoreSchema entity = 1;
  map<string, BrainEventPropertySchema> property = 2;
}
message BrainIotSensorEventSchema {
  BrainEntityStoreSchema entity = 1; // information about sensor id
  google.protobuf.Timestamp time_stamp = 2;
  map<string, BrainEventPropertySchema> property = 3;
}
message BrainEntityStoreSchema {
  // key = mid(/m/<vertical>/<domain>/entity/<entity-type>)
  map<string, jio.brain.proto.entity.BrainEntitySchema> entity = 1;
}

Feature definition and DAG
---
#### Step 1 :
Identify important features that will be required to run different machine learning models, rule engine or optimization. Also identify feature family.

message BrainEventFeatureSchema {
  jio.brain.proto.base.BrainToken token = 1;
  jio.brain.proto.quantity.BrainQuantitySchema quantity = 2;
  optional jio.brain.proto.base.BrainToken unit = 3;
  jio.brain.proto.base.BrainTimeWindowType time_window = 4;
}
message BrainFeatureFamilySchema {
  jio.brain.proto.base.BrainToken token = 1;   // feature_family_token
  repeated string features = 2;
}

#### Step 2 :
Using brain os building blocks create a pseudocode defining data transformation from ingestion to target features in form of DAG
##### Processes

message BrainProcessSchema {
  jio.brain.proto.base.BrainToken token = 1;
  string input_topic = 2;
  jio.brain.proto.event.BrainEventSchema input_event_schema = 3;
  map<string, jio.brain.proto.event.BrainEventSchema> output_event_schema = 4; // key = output_topic, value = event key in dictionary
  jio.brain.proto.base.BrainProcessType brain_process_type = 5;
  oneof config {
    BrainEnrichProcessSchema enrich = 6;
    BrainFilterProcessSchema filter = 7;
    BrainPivotProcessSchema pivot = 8;
    BrainReduceProcessSchema reduce = 9;
    BrainIngestProcessSchema ingest = 10;
    BrainQuantizeProcessSchema quantize = 11;
    BrainComputeProcessSchema compute = 12;
    BrainFunctionProcessSchema function = 13;
    BrainRepositoryProcessSchema repository = 14;
  }
}

#### Step 3 :
Define input and output event schema for each processes in DAG

message BrainEventKeySchema {
  BrainEntityStoreSchema entity = 1;
  // key = mid(/m/<vertical>/<domain>/<event>/<context>)
  map<string, BrainEventContextSchema> context = 2;
}
message BrainEventValueSchema {
  BrainEntityStoreSchema entity = 1;
  // key = mid(/m/<vertical>/<domain>/<event>/<property>)
  map<string, BrainEventPropertySchema> property = 2;
}

message BrainEventKeyValueSchema {
  BrainEventKeySchema key = 1;
  BrainEventValueSchema value = 2;
}
message BrainEventFeatureKeyValueSchema {
  BrainEventKeySchema key = 1;
  BrainEventFeatureValueSchema value = 2;
}

message BrainEventFeatureValueSchema {
  // key = mid(/m/<vertical>/<domain>/<event>/<property>)
  jio.brain.proto.feature.BrainFeatureSetSchema features = 1;
  jio.brain.proto.entity.BrainAttributeSetSchema attribute_schema = 2;
}

message BrainEntityStoreSchema {
  // key = mid(/m/<vertical>/<domain>/entity/<entity-type>)
  map<string, jio.brain.proto.entity.BrainEntitySchema> entity = 1;
}
message BrainEventContextSchema {
  jio.brain.proto.base.BrainToken token = 1;
  jio.brain.proto.quantity.BrainQuantitySchema quantity = 2;
  // BrainToken unit = 3;
}
message BrainEventPropertySchema {
  jio.brain.proto.base.BrainToken token = 1;
  jio.brain.proto.quantity.BrainQuantitySchema quantity = 2;
  optional jio.brain.proto.base.BrainToken unit = 3;
}

#### Step 4 :
Define configuration of each process in brain ui workbench
##### Enrich

message BrainEnrichProcessSchema {
  string output_topic = 1;
  EnrichAttributesInstance attribute = 2;
  EnrichPredicateInstance predicate = 3;
}

message EnrichAttributesInstance {
  map<string, jio.brain.proto.types.BrainStrVector> attribute  = 1;
}

message EnrichPredicateInstance {
  map<string, jio.brain.proto.types.BrainStrVector>  predicate = 1;
}

##### Filter

message BrainFilterProcessSchema {
  string input_topic = 1;
  repeated FilterInstance instances = 2;
  message FilterInstance {
    string filter_name = 1;
    string success_output_topic = 2;
    string failure_output_topic = 3;
    string filter_expression = 4;          // key.attribute.customer.type == "premium" && value.property.invoice.total > 1000
  }

}

##### Pivot

message BrainPivotProcessSchema {
  string input_topic = 1;
  repeated PivotInstance instances = 2;
}
message PivotInstance {
  string output_topic = 1;
  string output_event_topic = 2;
  BrainPivotKeyConfig key_config = 3;
  BrainPivotValueConfig value_config = 4;
}

message BrainPivotKeyConfig {
  repeated string entities = 1;
  repeated string attributes = 2;
  map<string, string> contexts = 3; 
}

message BrainPivotValueConfig {
  repeated string entities = 1;     
  repeated string attributes = 2;
  map<string, string> properties = 3;
}

##### Reduce

message BrainReduceProcessSchema {
  repeated ReduceInstance instances = 1;
}
message ReduceInstance {
  repeated AggregateCondition aggregate_condition = 1;
  string output_topic = 2;
  string output_event_schema = 3;
  optional jio.brain.proto.quantity.BrainTimeInstance start_time = 4;
  optional jio.brain.proto.quantity.BrainTimeInstance end_time = 5;
  uint32 window_time_minutes = 6;  // 10-10:30  , 10:15-10:45
  jio.brain.proto.base.BrainTimeWindowType  time_window = 7;
}
message AggregateCondition {
  optional string input_property = 1;
  optional string output_property = 2;
  optional string count_property = 3;
  AggregateConditionType type = 4;
  enum AggregateConditionType {
    BRAIN_AGGREGATION_SUM = 0;
    BRAIN_AGGREGATION_AVERAGE = 1;
    BRAIN_AGGREGATION_COUNT = 2;
    BRAIN_AGGREGATION_MIN = 3;
    BRAIN_AGGREGATION_MAX = 4;
    BRAIN_AGGREGATION_SLOPE = 5;
    BRAIN_AGGREGATION_PRODUCT = 6;
    BRAIN_AGGREGATION_DISTRIBUTION = 7;
  }

}

##### Compute

message BrainComputeProcessSchema {
  string output_event_schema = 1;
  string output_topic = 2;
  repeated ComputeInstance instances = 3;
}
message ComputeInstance {
  FieldType type = 1;
   string compute_expression = 2;  // (product.mrp - product.discount) * product.quantity + product.gst  ( BODMAS )
   string output_property_name=3;  // product.price or bill_amount
  enum FieldType {
      attribute=0;
      property=1;
  }
}

##### Quantize

message BrainQuantizeProcessSchema {
  string output_event_schema = 1;
  string output_topic = 2;
  repeated QuantizeInstance instances = 3;
}
message QuantizeInstance {
  InputPropertyType type = 1;
  string input_property = 2;
  string output_property = 3;

  enum InputPropertyType {
    BRAIN_QUANTIZE_TEMPORAL = 0;
    BRAIN_QUANTIZE_SPATIAL = 1;
    BRAIN_QUANTIZE_BINNED = 2;
  }
}

###### tags: `onboarding` `schema`