# DL and ML Bridge

###### tags: `ml`

## Image Classification For Agriculture
> Although agriculture use case us central to it, but implementation is indepednent of any domain
### Sequence Diagram

ImageInputTopic->ImageEventBridge: receive
Note right of ImageEventBridge: Get image from event
ImageEventBridge->EventPreprocessor: pre process image
ImageEventBridge->ImagePreprocessor: pre process image
ImageEventBridge->MlModel: infer
ImageEventBridge->EventPostProcessor: trasform result
ImageEventBridge->ClaasificationOutputTopic: trasform result

### Events
#### Parent Event

message BrainEvent {
  uint32 id = 1;
  uint32 type = 2;
  BrainEventMetadata meta_data = 3;
  oneof event {
    BrainIngestionEvent ingestion_event = 4;
    BrainTransactionEvent transaction_event = 5;
    BrainKeyValueEvent key_value_event = 6;
    BrainEventFeatureKeyValue feature_event = 7;
    BrainMultivariateEvent multivariate_event = 8;
    BrainIotEvent iot_event = 9;
    BrainImageEvent image_event = 10;
    BrainImageClassificationEvent image_classification = 11;
  }
}
message BrainEventSchema {
  jio.brain.proto.base.BrainToken token = 1;
  oneof event_schema {
    BrainIngestionEventSchema ingestion_event = 2;
    BrainTransactionEventSchema transaction_event = 3;
    BrainEventKeyValueSchema key_value_event = 4;
    BrainEventFeatureKeyValueSchema feature_event = 5;
    BrainMultivariateEventSchema multivariate_event = 6;
    BrainIotEventSchema iot_event = 7;
    BrainImageEventSchema image_event = 8;
    BrainImageClassificationEventSchema image_classification_event = 9;
  }
}

#### Input Event for Agriculture use case

message BrainImageEvent {
  string source_uri = 1;
  oneof payload_is_one_of {
    string uri = 2; // Uri in content store of image
    Bytes bytes = 3;
    string base64 = 4; // base64 image
  }
  string processed_payload = 5; // file path on local disk of pre processed image or persistent volume. Its name will be generated using pre processor and image id
  BrainEntityStore entity_store = 6; // farmer id
  BrainQuantityStore context_store = 7; // location and time
  BrainQuantityStore property_store = 8; // other properties
}
message Bytes {
  repeated bytes value = 1;
  string content_type = 2;
}
message BrainImageEventSchema {
  BrainImagePayloadType payload_type = 1;
  BrainEntityStoreSchema entity = 2;
  map<string, BrainEventContextSchema> context = 3; // key = mid(/m/<vertical>/<domain>/<event>/<context>)
  map<string, BrainEventPropertySchema> property = 4; // key = mid(/m/<vertical>/<domain>/<event>/<property>)
}
enum BrainImagePayloadType {
  BRAIN_IMAGE_PAYLOAD_TYPE_URI = 0;
  BRAIN_IMAGE_PAYLOAD_TYPE_BYTES = 1;
  BRAIN_IMAGE_PAYLOAD_TYPE_BASE64 = 2;
}

#### Output Event for Agriculture use case

message BrainImageClassificationEvent {
  uint32 model_id = 1; // identify it is inference of which model
  BrainImageEvent source_event = 2; // input event
  repeated BrainImageLabel image_label_set = 3; // key is class (symbol or entity) and value is probability (between 0 and 1)
}
message BrainImageLabel {
  uint64 brain_id  = 1; // entity id, or symbol id
  uint32 x = 2;
  uint32 y = 3;
  uint32 length = 4;
  uint32 width = 5;
}
message BrainImageClassificationEventSchema {
  BrainImageEventSchema source_event_schema = 1;
  repeated BrainImageLabelSchema image_label_set = 2;
  map<string, string> meta_data = 3; // type =  (x-ray, ct-scan, mri, spectral) , sub type = (knee, shoulder, etc)
}
message BrainImageLabelSchema {
  jio.brain.proto.quantity.BrainSymbolicQuantitySchema symbolic = 2;
}

## Deployment
### Services type
**Static Services :** Inference Lake, Function Resolver, Function Router
**Dynamic Services :** Azure Ingestion Service, Deep Learning Bridge, Pre Processor, Ensembler

### Static Services
Always runnung generic services part of Brain OS.
1. Inference Lake - Stores all output of ML and DL model
2. Function Resolver - Identify which function to execute bases on Inference update Event, Feature update event (result of DAG) and Knowledge update event
3. Function Router - Based on output of function resolver (or feature resolver in case of multivariate event) - Create required event for function and route event to required topic

### Dynamic Services
Brain OS managed dynamically deployed services based on process configuration
#### Azure Ingestion Service
Service listen to azure bucket and create an image event

message BlobStorageTransport {
  string bucket_name = 1;
  string prefix = 2;
  string suffix = 3;
  string connection_key = 4;
}

#### Image Bridge
It will consist of three runing containers
1. Image Bridge
2. Image Pre Processor
3. ML Flow Process

> Post processor processing will be embedded in image bridge


##### Deployment Yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: image-bridge
spec:
  selector:
    matchLabels:
      app: image-bridge
  replicas: 1
  template: # template for the pods
    metadata:
      labels:
        app: image-bridge
    spec:
      volumes:
        - name: image-folder
          persistentVolumeClaim:
            claimName: image-storage-pvc
      containers:
        - name: image-bridge
          image: jioaicr.azurecr.io/brain-os/image-bridge:latest
          volumeMounts:
            - name: image-folder
              mountPath: /mnt/images
          env:
            - name: KAFKA_HOST
              value: 10.168.135.9:9092,10.168.134.148:9092,10.168.134.247:9092
            - name: CONFIG_URL
              value: http://config-service:8080/process/
            - name: PROCESS_KEY
              value: <PROCESS_KEY>
            - name: PRE_PROCESSOR_URL
              value: http://localhost:8080
            - name: PRE_PROCESSOR_FOLDER
              value: /mnt/images/pre_process
            - name: INFERENCE_URL
              value: http://localhost:9090
            - name: DELETE_IMAGE_AFTER_PROCESSING
              value: false
        - name: image-preprocessor
          image: jioaicr.azurecr.io/brain-os/image-pre-processor:latest
          volumeMounts:
            - name: image-folder
              mountPath: /mnt/images
          env:
            - name: CONFIG_URL
              value: http://config-service:8080/process/
            - name: PROCESS_KEY
              value: <PROCESS_KEY>
            - name: PRE_PROCESSOR_FOLDER
              value: /mnt/images/pre_process
        - name: image-model
          image: jioaicr.azurecr.io/brain-os/crop-classification-model:latest
          volumeMounts:
            - name: image-folder
              mountPath: /mnt/images
          env:
            - name: CONFIG_URL
              value: http://config-service:8080/process/
            - name: PROCESS_KEY
              value: <PROCESS_KEY>
            - name: PRE_PROCESSOR_FOLDER
              value: /mnt/images/pre_process
            

### Process Schema

message BrainFunctionSchema {
  jio.brain.proto.base.BrainToken token = 1;
  map<string, jio.brain.proto.event.BrainEventSchema> input_event_schema = 2;
  map<string, jio.brain.proto.event.BrainEventSchema> output_event_schema = 4; // key = output_topic, value = event key in dictionary
  jio.brain.proto.base.BrainProcessType brain_process_type = 5;
  oneof config {
    BrainEnrichFunctionSchema enrich = 6;
    BrainFilterFunctionSchema filter = 7;
    BrainPivotFunctionSchema pivot = 8;
    BrainReduceFunctionSchema reduce = 9;
    BrainIngestFunctionSchema ingest = 10;
    BrainQuantizeFunctionSchema quantize = 11;
    BrainComputeFunctionSchema compute = 12;
    BrainJoinerFunctionSchema joiner = 13;
    BrainImageFunctionSchema image_bridge = 14;
    BrainMultivariateBridgeFunctionSchema multivariate_bridge = 15;
  }
}
message BrainImageFunctionSchema {
  string output_topic = 1;
  string output_event_schema = 2;
  string model_key = 3; // Delete local images if this is last model
  string pre_requisite_model_token = 4; // help creating model DAG. If present, this function will be called only if pre requisite model is called.
  string model_docker_image = 5; // MLFlow generated docker image for image classification
  uint32 port = 6; // model api port
  BrainImagePreprocessorConfig image_pre_processor_config = 7;
  BrainImagePostProcessorConfig post_processor_config = 8;
  string local_path = 9; // path to keep intermediate image data
}

Above model_docker_image is mlflow enabled docker image

#### Image Pre Processor

message BrainImageFunctionSchema {
  string output_topic = 1;
  string output_event_schema = 2;
  string model_key = 3; // Delete local images if this is last model
  string pre_requisite_model_token = 4; // help creating model DAG. If present, this function will be called only if pre requisite model is called.
  string model_docker_image = 5; // MLFlow generated docker image for image classification
  uint32 port = 6; // model api port
  BrainImagePreprocessorConfig image_pre_processor_config = 7;
  BrainImagePostProcessorConfig post_processor_config = 8;
  string local_path = 9; // path to keep intermediate image data
}
message BrainImagePreprocessorConfig {
  map<uint32, BrainImagePreProcessor> pre_processor_steps = 1; // key is step number, value is non repeatable BrainImagePreProcessor
}
message BrainImagePreProcessor {
  oneof pre_processor {
    BrainImageResizer resizer = 1;
    BrainImageColorConversion color_conversion = 2;
    BrainImageCropping cropping = 3;
    BrainImageAdjustment adjustment = 4;
    BrainImageTranspose transpose = 5;
  }
}
message BrainImageResizer {
  uint32 height = 1;
  uint32 width = 2;
  uint32 padding = 3;
}
message BrainImageColorConversion {
  BrainImageColorConversionType source = 1;
  BrainImageColorConversionType target = 2;
}
enum BrainImageColorConversionType {
  BRAIN_IMAGE_COLOR_TYPE_RGB = 0;
  BRAIN_IMAGE_COLOR_TYPE_GRAY_SCALE = 1;
  BRAIN_IMAGE_COLOR_TYPE_HSV = 2;
  BRAIN_IMAGE_COLOR_TYPE_YIQ = 3;
  BRAIN_IMAGE_COLOR_TYPE_YUV = 4;
}
message BrainImageCropping {
  uint32 x_coordinate = 1;
  uint32 y_coordinate = 2;
  uint32 height = 3;
  uint32 width = 4;
}
message BrainImageAdjustment {
  // TBD
}
message BrainImageTranspose {
  // TBD
}

#### Image Post Processor
> It is part of bridge process

message BrainImagePostProcessorConfig {
  oneof post_processor_is_one_of {
      BrainImageClassificationPostProcessorConfig classification_post_processor = 1;
  }
}
/*
Labeling an x-ray as cancer or not (binary classification).
Classifying a handwritten digit (multiclass classification).
Assigning a name to a photograph of a face (multiclass classification).
 */
message BrainImageClassificationPostProcessorConfig {
  string entity_type = 1;
  repeated BrainImageLabel image_label_set = 1;
}
message BrainImageLabel {
  uint64 brain_id = 1;
  jio.brain.proto.quantity.BrainRangeQuantity range = 2;
}

### Ensembler
> currently we will support only **majority wins** ensembler

Result will be matched on unique **event id**, and it is assumed that result should get achieved in **window_time_minutes**

message BrainEnsemblerFunctionSchema {
  string output_event_schema = 1;
  string output_topic = 2;
  uint32 window_time_minutes = 6;  
  oneof ensembler_is_one_of {
    BrainEnsemblerMajority majority = 8;
    BrainEnsemblerBagging bagging = 9;
    BrainEnsemblerStacking stacking = 10;
    BrainEnsemblerBoosting boosting = 11;
  }
}
message BrainEnsemblerMajority {
  // TBD
}
message BrainEnsemblerBagging {
  // TBD
}
message BrainEnsemblerStacking {
  // TBD
}
message BrainEnsemblerBoosting {
  // TBD
}

**Ensembler** - machine learning technique that combines several base models in order to produce one optimal predictive model. https://machinelearningmastery.com/tour-of-ensemble-learning-algorithms/

1. Majority
2. BAGGing, or Bootstrap AGGregating
3. Stacking
4. Boosting

## Multivariate Model
TBD