23siddhi first met

Time:2022-3-24

siddhi-base

  • Event driven: the data consumed, processed and sent are regarded as one event

  • Stream processing and complex event processing platform

  • Siddhi application is a script similar to SQL Semicolons in siddhi scripts indicate the end of a sentence.

  • Including consumers, producers, streams, queries, tables and methods, and other necessary conventions

  • It can accept \ send to many different types of event input / output, such as TCP, HTTP, Kafka, file, etc.

  • It can be accepted and transformed into different data formats: JSON, text, XML, key value

  • Handle events, transform and analyze

  • Process: accept the event for consumption, pass the event to the corresponding query processing, form a new event according to the logic, and send the new event to the stream

  • Stream and Query

    • define stream InputTemperatureStream (sensorId string, temperature double);--  Define a flow. The parameters are the sensor ID of string and the temperature of double
      @Info (name = 'pass through') -- optional annotation of query
      from InputTemperatureStream
      select *
      insert into TemperatureAndSensorStream; --  Consume data from inputtemperaturestream, obtain all attributes through select *, produce new events, and output them to temperatureandsensorstream
      @info(name = 'Simple-selection')
      From temperatureandsensorstream -- consume events from temperatureandsensorstream
      select temperature
      insert into TemperatureOnlyStream;--  Output temperature only to temperatureonlystream
      --Events with a value of ['aq-14 ', 35.4] are input to inputtemperaturestream
      -- InputTemperatureStream : ['aq-14', 35.4]
      -- TemperatureAndSensorStream : ['aq-14', 35.4]
      -- TemperatureOnlyStream : [35.4]
    • A total of @ info is used to annotate the keywords define stream, from, select and insert into

  • Source and Sink

    • @source(type='http', receiver.url='http://0.0.0.0:8006/temp',@map(type='json'))
      --The data accepted by the consumer is HTTP, and the address is JSON
      define stream TemperatureStream (sensorId string, temperature double);
      --Define stream temperaturestream
      @Sink (type ='log ') -- the producer output stream is log
      @sink(type='kafka',topic='temperature',bootstrap.servers='localhost:9092',@map(type='json',@payload("""{"temp":"{{temperature}}"}""")))
      --The producer output stream is Kafka, the output is JSON, and the topic is temperature bootstrap The value of servers temperature is mapped to a JSON named temp
      define stream TemperatureOnlyStream (temperature double); --  Define output stream
      @info(name = 'Simple-selection') 
      from TemperatureStream 
      select temperature 
      insert into TemperatureOnlyStream;
      --Query queries the temperature attribute from temperaturestream and inputs it into temperatureonlystream
      
      
      --Send a message of type input JSON to http://0.0.0.0:8006/temp When, an event will occur
      {
         "event":{
            "sensorId":"aq-14",
            "temperature":35.4
         }
      }
      --After data processing, the event will arrive at the stream temperatureonlystream and be sent to the temperatureonlystream stream through log and Kafka
      --Log is produced and output to the console through passthrough
      Event{timestamp=1574515771712, data=[35.4], isExpired=false}
      --Kafka maps the stream to JSON and sends it to temperature topic
      {"temp":"35.4"}
    • A total of @ source, @ sink, @ map, @ info [Kafka] [@ payload] annotations are used

  • Table and Store

    • define stream TemperatureStream (sensorId string, temperature double); -- define stream
      define table TemperatureLogTable (sensorId string, roomNo string, temperature double); -- define stream
      @store(type="rdbms",jdbc.url="jdbc:mysql://localhost:3306/sid",username="root", password="root",jdbc.driver.name="com.mysql.jdbc.Driver")
      --Store to relational database connect to the database specified for user name password driver name
      define table SensorIdInfoTable (sensorId string, roomNo string); --  The definition table is stored in memory
      @info(name = 'Join-query')
      from TemperatureStream as t join SensorIdInfoTable as s
           on t.sensorId == s.sensorId
      --Define the query stream join table on keyword to be equal
      select t.sensorId as sensorId, s.roomNo as roomNo, t.temperature as temperature
      insert into TemperatureLogTable;
      --Store the Cartesian product query results into the table
      
      --When the sensoridinfotable contains a record ['aq-14 ',' 789 '], and when an event ['aq-14', 35.4] reaches the temperaturestream, an event will be generated, converted to ['aq-14 ',' 789 ', 35.4], and added to the temperaturelogtable.
      
      --Data in query table
      -- on-demand query 
      from TemperatureDetailsTable
      select *
      -- calling query() method of SiddhiAppRuntime
    • A total of @ store and 1 annotation table keyword are used

  • Siddhi Application

    • @app:name('Temperature-Processor') -- siddhi app name
      @app:description('App for processing temperature data.') -- app description 
      @source(type='inMemory', topic='SensorDetail') 
      define stream TemperatureStream (sensorId string, temperature double);
      --The consumer reads the event definition stream of other apps from memory
      @sink(type='inMemory', topic='Temperature')
      define stream TemperatureOnlyStream (temperature double);
      --The producer publicly publishes the event definition stream to other apps in memory
      @info(name = 'Simple-selection')
      from TemperatureStream
      select temperature
      insert into TemperatureOnlyStream;
      --Define query to automatically query the temperature from the consumer, pass the event to the producer, and the producer will disclose the event
      --['aq-14 ', 35.4] received from topic sensordetail will be passed to temperaturestream
      --[35.4] arrive at temperatureonlystream and transfer it to other subscribed apps in memory through topic temperature
    • A total of two keywords @ app: name @ app: description are used

  • Basic Types:int long float double string object

    • convert() instanceof()

    • Objects are divided into list, map, etc

    • define stream PatientRegistrationInputStream (
                       seqNo long, name string, age int,
                       height float, weight double, photo object,
                       isEmployee bool, wardNo object);
      
      define stream PatientRegistrationStream (
                       seqNo long, name string, age int,
                       height double, weight double, photo object,
                       isPhotoString bool, isEmployee bool,
                       wardNo int);
      --Define two streams with different data types for conversion
      @info(name = 'Type-processor')
      from PatientRegistrationInputStream
      select seqNo, name, age,
             convert(height, 'double') as height,
      
             weight, photo,
             instanceOfString(photo) as isPhotoString,
      
             isEmployee,
             cast(wardNo, 'int') as wardNo
             
      --Convert (value, to_type) returns the value after converting the type 
      --Instanceofstring (value) Returns whether it belongs to string type
      --Cast (value, to_type) returns the value after type conversion 
      --Convert is an explicit conversion and cast is a strong conversion
      
      insert into PatientRegistrationStream;
      
      --[1200098, 'Peter Johnson', 34, 194.3f, 69.6, #fjoiu59%3hkjnknk $#nft, true, 34] is the patientregistrationinputstream event
      -- [1200098, 'Peter Johnson', 34, 194.3, 69.6, #Fjoiu59%3hkjnknk$#nFT, false, true, 34] PatientRegistrationStream  event
    • Four methods, convert() instanceofxxx() cast() convert() are used

  • Map

    • define stream CoupleDealInfoStream (item1 string, price1 double,item2 string, price2 double);
      @info(name = 'Create-map')
      from CoupleDealInfoStream
      select map:create(item1, price1, item2, price2)
      as itemPriceMap
      -- query map:create(key1,value1,...,...)  Create a map key value pair as I1: P1 and I2: P2
      insert into NewMapStream;
      
      
      @info(name = 'Check-map')
      from NewMapStream
      select map:isMap(itemPriceMap) as isMap,
      
             map:containsKey(itemPriceMap, 'Cookie')
                  as isCookiePresent,
      
             map:containsValue(itemPriceMap, 24.0)
                  as isThereItemWithPrice24,
      
             map:isEmpty(itemPriceMap) as isEmpty,
      
             map:keys(itemPriceMap) as keys,
      
             map:size(itemPriceMap) as size
      insert into MapAnalysisStream;
      --Map: ismap (map) determines whether it is a map
      --Map: containskey (map, key) determines whether there is a key
      --Map: containsvalue (map, value) determines whether there is this value
      --Map: isempty (map) determines whether the map is empty
      --Map: keys (map) returns keyset
      --Map: size (map) returns the size of the map
      
      @info(name = 'Clone-and-update')
      from NewMapStream
      select map:replace(
                         map:put(map:clone(itemPriceMap),
                                 "Gift",
                                 1.0),
                         "Cake",
                         12.0) as itemPriceMap
      
      insert into ItemInsertedMapStream;
      --Map: replace (map, key, value) replace the value of the key in the map with a new value 
      --Map: put (map, key, value) adds a key value pair to the map and returns the map
      --Map: Clone (map) returns a cloned map
      
      
      --Coupledealinfostream received event ['chocolate ', 18.0,' ice cream ', 24.0]
      -- NewMapStream [{Ice Cream=24.0, Chocolate =18.0}]
      -- MapAnalysisStream [true, false, true, false, [Ice Cream, Chocolate], 2]
      -- ItemInsertedMapStream [{Ice Cream=12.0, Gift=1.0, Chocolate =18.0}]
  • List

    • define stream ProductComboStream (product1 string, product2 string, product3 string);
      
      @info(name = 'Create-list')
      from ProductComboStream
      select list:create(product1, product2, product3)
                  as productList
      
      insert into NewListStream;
      -- list:create(item1,item2,...) 
      @info(name = 'Check-list')
      from NewListStream
      select list:isList(productList) as isList,
      
             list:contains(productList, 'Cake')
                  as isCakePresent,
      
             list:isEmpty(productList) as isEmpty,
      
             list:get(productList, 1) as valueAt1,
      
             list:size(productList) as size
      
      insert into ListAnalysisStream;
      --List: islist (list) determines whether it is a list
      --List: contains (list, item) determines whether it contains elements
      --List: isempty (list) determines whether it is empty
      --List: get (list, index) to get subscript elements
      --List: size (list) returns the size of the list
      
      @info(name = 'Clone-and-update')
      from NewListStream
      select list:remove(
                  list:add(list:clone(productList), "Toffee"),
                  "Cake") as productList
      
      insert into UpdatedListStream;
      --List: remove (list, item) deletes an element
      --List: add (list, item) adds elements and returns list
      --List: Clone (list), which returns the list of clones
      
      --Productcombostream add event ['ice cream ',' chocolate ',' cake ']
      -- NewListStream [[Ice Cream, Chocolate, Cake]] 
      -- ListAnalysisStream [true, true, false, Chocolate, 3]
      -- UpdatedListStream [[Ice Cream, Chocolate, Toffee]]
  • Null

    • define stream ProductInputStream (item string, price double);
      define Table ProductInfoTable (item string, discount double);
      
      @info(name = 'Check-for-null')
      from ProductInputStream [not(item is null)]
      --Not (test) returns the result of the expression true / false 
      --[] is equivalent to if judgment condition
      --Object is null returns whether it is null
      select item,
             price is null as isPriceNull
      
      insert into ProductValidationStream;
      
      @info(name = 'Outer-join-with-table')
      from ProductInputStream as s
          left outer join ProductInfoTable as t
          on s.item == t.item
      select s.item, s.price, t.discount,
             math:power(t.discount, 2) is null
                  as isFunctionReturnsNull,
      
             t is null as isTNull,
             s is null as isSNull,
      
             t.discount is null as isTDiscountNull,
             s.item is null as isSItemNull
      
      insert into DiscountValidationStream;
      --Math: power (number, exp) power operation
      -- ProductInputStream ['Cake', 12.0]
      -- ProductValidationStream [Cake, false]
      -- DiscountValidationStream [Cake, 12.0, null, true, true, false, true, false]

siddhi-event&data

event cleansing

  • value based filtering

    • define stream TemperatureStream (
       sensorId string, temperature double);
      
      @info(name = 'EqualsFilter')
      from TemperatureStream[ sensorId == 'A1234']
      select *
      insert into SenorA1234TemperatureStream;
      --[] judge if
      
      @info(name = 'RangeFilter') 
      from TemperatureStream[ temperature > -2 and temperature < 40]
      select *
      insert into NormalTemperatureStream;
      -- > < and 
      
      @info(name = 'NullFilter') 
      from TemperatureStream[ sensorId is null ]
      
      select *
      insert into InValidTemperatureStream;
      -- is null
    • [] judge if > < and is null

  • if-then-else

    • define stream TemperatureStream 
              (sensorId string, temperature double);
      
      @info(name = 'SimpleIfElseQuery')
      from TemperatureStream
      select sensorId,
       ifThenElse(temperature > -2, 'Valid', 'InValid') as isValid 
      --Ifthenelse (test, truevalue, false value) ternary expression
      insert into ValidTemperatureStream;
      
      
      @info(name = 'ComplexIfElseQuery') 
      from TemperatureStream
      select sensorId, 
       ifThenElse(temperature > -2, 
              ifThenElse(temperature > 40, 'High', 'Normal'), 
              'InValid') 
          as tempStatus
      --Nested binocular expression
      insert into ProcessedTemperatureStream;
    • Ifthenelse (test, truevalue, false value) ternary expression

  • regex matching

    • define stream SweetProductionStream (name string, amount int);
      
      @info(name='ProcessSweetProductionStream')
      from SweetProductionStream
      select name, 
         regex:matches('chocolate(.*)', name) as isAChocolateProduct, 
         regex:group('.*\s(.*)', name, 1) as sweetType
      --Regex: matches (regex, string) determines whether the string matches the pattern and returns true / false
      --Regex: Group (regex, string, index) after string matches regex, it is divided into groups and returns the element with index as the subscript
      -- '.* \S (. *) 'any character with a space in the middle
      insert into ChocolateProductStream; 
      
      -- SweetProductionStream ['chocolate cake', 34]
      -- ChocolateProductStream ['chocolate cake', true, 'cake']
  • default

    • define stream PatientRegistrationInputStream (
                       seqNo long, name string, age int,
                       height float, weight double, photo object,
                       isEmployee bool, wardNo object);
      
      @info(name = 'SimpleIfElseQuery')
      from PatientRegistrationInputStream
      select 
       default(name, 'invalid') as name, 
       default(seqNo, 0l) as seqNo, 
       default(weight, 0d) as weight,
       default(age, 0) as age, 
       default(height, 0f) as height   
      --Default (param, DefaultValue) if the value does not exist, it is the default value
      insert into PreprocessedPatientRegistrationInputStream;
      --Default value of each type ['Invalid ', 0.0, 0, 0.0]
  • type based filtering

    • define stream SweetProductionStream (name string, amount int);
      
      @info(name='ProcessSweetProductionStream')
      from SweetProductionStream
      select 
         instanceOfInteger(amount) as isAIntInstance,
      
          name, 
          amount
      insert into ProcessedSweetProductionStream;
      --Put the result of judgment type into event as filter
  • remove duplicate event

    • define stream TemperatureStream
              (sensorId string, seqNo string, temperature double);
              
      @info(name = 'Deduplicate-sensorId')
      from TemperatureStream#unique:deduplicate(sensorId, 1 min)
      
      -- stream_ Name#unique: duplicate (ID, time_gap) de duplicates the events that arrive between time gaps according to the ID, and those beyond the gap are not counted as duplicates
      
      select *
      insert into UniqueSensorStream;
      @info(name = 'Deduplicate-sensorId-and-seqNo')
      from TemperatureStream#unique:deduplicate(
                              str:concat(sensorId,'-',seqNo), 1 min)
      -- str:concat(str1,concat_symbol,str2) 
      select *
      insert into UniqueSensorSeqNoStream;

data transformation

  • math & logical operation

    • define stream TemperatureStream
              (sensorId string, temperature double);
      @info(name = 'celciusTemperature')
      from TemperatureStream
      select sensorId, 
                      (temperature * 9 / 5) + 32 as temperature
      
      insert into FahrenheitTemperatureStream;
      @info(name = 'Overall-analysis')
      from FahrenheitTemperatureStream
      select sensorId, 
                      math:floor(temperature) as approximateTemp 
      --Math: floor (number) rounded down
      insert all events into OverallTemperatureStream;
      @info(name = 'RangeFilter') 
      from OverallTemperatureStream
                    [ approximateTemp > -2 and approximateTemp < 40]
      
      select *
      insert into NormalTemperatureStream;
  • transform json

    • define stream InputStream(jsonString string);
      
      from InputStream 
      select json:toObject(jsonString) as jsonObj 
      insert into PersonalDetails;
      -- json:toObject(string) json->object
      
      from PersonalDetails
      select jsonObj, 
          json:getString(jsonObj,'$.name') as name,
          json:isExists(jsonObj, '$.salary') as isSalaryAvailable,
          json:toString(jsonObj) as jsonString 
      --JSON: getString (object, "$. Item") returns the value corresponding to the item name
      --JSON: isexists (object, "$. Item") determines whether the value corresponding to the item name exists
      --JSON: toString (object) returns the string corresponding to the object
      insert into OutputStream;
      
      from OutputStream[isSalaryAvailable == false]
      select 
          json:setElement(jsonObj, '$', 0f, 'salary') as jsonObj
      --JSON: setelement (object, '$', value, 'item') assigns the value of item to value and adds it to object
      insert into PreprocessedStream;
      
      {
          "name" : "siddhi.user",
          "address" : {
              "country": "Sri Lanka",
          },
          "contact": "+9xxxxxxxx"
      }
      
      OutputStream
      [ {"address":{"country":"Sri Lanka"},"contact":"+9xxxxxxxx","name":"siddhi.user"}, siddhi.user, false,
      "{\"name\" : \"siddhi.user\", \"address\" : { \"country\": \"Sri Lanka\", }, \"contact\": \"+9xxxxxxxx\"}"]
      
      PreprocessedStream
      {
          "name" : "siddhi.user",
          "salary": 0.0
          "address" : {
              "country": "Sri Lanka",
          },
          "contact": "+9xxxxxxxx"
      }

data summarization

  • sliding time

    • define stream TemperatureStream
              (sensorId string, temperature double);
      
      @info(name = 'Overall-analysis')
      from TemperatureStream#window.time(1 min)
      # window. Time (time_gap) the time interval between sliding windows is less than 1 minute
      
      select avg(temperature) as avgTemperature,
             max(temperature) as maxTemperature,
             count() as numberOfEvents
      insert all events into OverallTemperatureStream;
      -- avg(number) 
      -- max(number)
      -- count() 
      --Insert all events into stream adds all events into the stream within the time interval and deletes them after expiration
      
      @info(name = 'SensorId-analysis')
      from TemperatureStream#window.time(30 sec)
      select sensorId,
             avg(temperature) as avgTemperature,
             min(temperature) as maxTemperature
      group by sensorId
      having avgTemperature > 20.0
      --Group by ID having condition
      insert into SensorIdTemperatureStream;
      --Add to sliding window after filtering
    • Stream #method can be regarded as a method that limits the behavior of stream, such as de duplication, sliding window aggregation, etc.

  • batch (tumbling) time

    • define stream TemperatureStream
              (sensorId string, temperature double);
      
      @info(name = 'Overall-analysis')
      from TemperatureStream#window.timeBatch(1 min)
      select avg(temperature) as avgTemperature,
             max(temperature) as maxTemperature,
             count() as numberOfEvents
             
      -- stream#window. Timebatch (time_gap) every time_ Gap time is regarded as a batch, starting from the first event to arrive
      insert into OverallTemperatureStream;
      
      @info(name = 'SensorId-analysis')
      from TemperatureStream#window.timeBatch(30 sec, 0)
      -- stream#window. Timebatch (time_gap, start_timepoint) every time_ Gap time is regarded as a batch, starting from the start time
      select sensorId,
             avg(temperature) as avgTemperature,
             min(temperature) as maxTemperature
      group by sensorId
      having avgTemperature > 20.0
      insert into SensorIdTemperatureStream;
    • The difference between window: time() and window: timebatch() is that the former writes to the stream from the beginning of the time interval, while the latter writes to the stream after the end of the time interval.

  • sliding event count

    • define stream TemperatureStream
              (sensorId string, temperature double);
      
      @info(name = 'Overall-analysis')
      from TemperatureStream#window.length(4)
      -- stream#window. Length (len) the sliding window is limited by the length and aggregates the last len events
      select avg(temperature) as avgTemperature,
             max(temperature) as maxTemperature,
             count() as numberOfEvents
      insert into OverallTemperatureStream;
      
      @info(name = 'SensorId-analysis')
      from TemperatureStream#window.length(5)
      select sensorId,
             avg(temperature) as avgTemperature,
             min(temperature) as maxTemperature
      group by sensorId
      having avgTemperature >= 20.0
      insert into SensorIdTemperatureStream;
  • batch(tumbling) event count

    • define stream TemperatureStream
              (sensorId string, temperature double);
      
      @info(name = 'Overall-analysis')
      from TemperatureStream#window.lengthBatch(4)
      -- stream#window. The lengthbatch (len) sliding window is limited by the length, and the last len events are aggregated as batch
      select avg(temperature) as avgTemperature,
             max(temperature) as maxTemperature,
             count() as numberOfEvents
      insert into OverallTemperatureStream;
      
      @info(name = 'SensorId-analysis')
      from TemperatureStream#window.lengthBatch(5)
      select sensorId,
             avg(temperature) as avgTemperature,
             min(temperature) as maxTemperature
      group by sensorId
      having avgTemperature >= 20.0
      insert into SensorIdTemperatureStream;
  • session

    • define stream PurchaseStream
              (userId string, item string, price double);
      
      @info(name = 'Session-analysis')
      from PurchaseStream#window.session(1 min, userId)
      # stream#window. Session (time_gap, session_id) aggregates events through ID, and_ Gap time
      select userId,
             count() as totalItems,
             sum(price) as totalPrice
      group by userId
      insert into UserIdPurchaseStream;
      
      @info(name = 'Session-analysis-with-late-event-arrivals')
      from PurchaseStream#window.session(1 min, userId, 20 sec)
      --Aggregate events on the userid based sessionwindow. The session interval is 1 minute, allowing a delay of 20 seconds to capture the delayed arrival.
      select userId,
             count() as totalItems,
             sum(price) as totalPrice
      group by userId
      insert into OutOfOrderUserIdPurchaseStream;
  • named window

    • define stream TemperatureStream
              (sensorId string, temperature double);
      
      define window OneMinTimeWindow
              (sensorId string, temperature double) time(1 min) ;
      -- define window name() time_type(time_gap)
      @info(name = 'Insert-to-window')
      from TemperatureStream
      insert into OneMinTimeWindow;
      
      @info(name = 'Min-max-analysis')
      from OneMinTimeWindow
      select min(temperature) as minTemperature,
             max(temperature) as maxTemperature
      insert into MinMaxTemperatureOver1MinStream;
      
      @info(name = 'Per-sensor-analysis')
      from OneMinTimeWindow
      select sensorId,
             avg(temperature) as avgTemperature
      group by sensorId
      insert into AvgTemperaturePerSensorStream;

data pipelining

  • stream join

    • define stream TemperatureStream
              (roomNo string, temperature double);
      
      define stream HumidityStream
              (roomNo string, humidity double);
      
      @info(name = 'Equi-join')
      from TemperatureStream#window.unique:time(roomNo, 1 min) as t
          join HumidityStream#window.unique:time(roomNo, 1 min) as h
          on t.roomNo == h.roomNo
          
      -- stream#window. Unique: time (ID, time_gap) leaves only the unique events in the time interval
      -- stream join
      select t.roomNo, t.temperature, h.humidity
      insert into TemperatureHumidityStream;
      
      @info(name = 'Join-on-temperature')
      from TemperatureStream as t
          left outer join HumidityStream#window.time(1 min) as h
          on t.roomNo == h.roomNo
      --Events within 1 minute of stream t left outer connection stream H
      select t.roomNo, t.temperature, h.humidity
      insert into EnrichedTemperatureStream;
  • partition events by value

    • define stream LoginStream
              ( userID string, loginSuccessful bool);
      
      @purge(enable='true', interval='10 sec',
             idle.period='1 hour')
      partition with ( userID of LoginStream )
      [email protected] purge every 10 seconds, check the partition instances that have not received events within 1 hour and clear them
      --Partition partition by userid
      begin
          @info(name='Aggregation-query')
          from LoginStream#window.length(3)
          select userID, loginSuccessful, count() as attempts
          group by loginSuccessful
          insert into #LoginAttempts;
      --#loginatempts is the internal stream of the partition and can only be accessed internally 
      
          @info(name='Alert-query')
          from #LoginAttempts[loginSuccessful==false and attempts==3]
          select userID, "3 consecutive login failures!" as message
          insert into UserSuspensionStream;
      end;
  • scatter and gather(String)

    • define stream PurchaseStream
                      (userId string, items string, store string);
      
      @info(name = 'Scatter-query')
      from PurchaseStream#str:tokenize(items, ',', true)
      select userId, token as item, store
      insert into TokenizedItemStream;
      --STR: tokenize (items, seperator, Boolean) items are connected with separators
      
      @info(name = 'Transform-query')
      from TokenizedItemStream
      select userId, str:concat(store, "-", item) as itemKey
      insert into TransformedItemStream;
      -- str:concat()
      @info(name = 'Gather-query')
      from TransformedItemStream#window.batch()
      select userId, str:groupConcat(itemKey, ",") as itemKeys
      insert into GroupedPurchaseItemStream;
      --STR: groupconcat (items, seperator) items are connected with separators
  • scatter and gather(json)

    • define stream PurchaseStream
                      (order string, store string);
      
      @info(name = 'Scatter-query')
      from PurchaseStream#json:tokenize(order, '$.order.items')
      --JSON: tokenize (JSON, '$. Name') accesses the values under name and separates them
      Select JSON: getString (order, '$. Order. ID') as OrderID, -- take out order id
             jsonElement as item,
             store
      insert into TokenizedItemStream;
      
      @info(name = 'Transform-query')
      from TokenizedItemStream
      select orderId,
             ifThenElse(json:getString(item, 'name') == "cake",
                        json:toString(
                          json:setElement(item, 'price',
                            json:getDouble(item, 'price') - 5
                          )
                        ),
                        item) as item,
             store
      insert into DiscountedItemStream;
      
      @info(name = 'Gather-query')
      from DiscountedItemStream#window.batch()
      select orderId, json:group(item) as items, store
      insert into GroupedItemStream;
      --JSON: Group (item) returns an item JSON array
      
      @info(name = 'Format-query')
      from GroupedItemStream
      select str:fillTemplate("""
          {"discountedOrder":
              {"id":"{{1}}", "store":"{{3}}", "items":{{2}} }
          }""", orderId, items, store) as discountedOrder
      insert into DiscountedOrderStream;
      --STR: filltemplate ({{n}}) is a placeholder similar to format (), which returns a spelled string

siddhi-others

  • Simple pattern one or more events arrive over time

    • define stream TemperatureStream(roomNo int, temp double);
      
      @sink(type = 'log')
      define Stream HighTempAlertStream(roomNo int,
          initialTemp double, finalTemp double);
      
      @info(name='temperature-increase-identifier')
      from every( e1 = TemperatureStream ) ->
          e2 = TemperatureStream[ e1.roomNo == roomNo
              and (e1.temp + 5) <= temp ]
          within 10 min
      select e1.roomNo, e1.temp as initialTemp, e2.temp as finalTemp
      insert into HighTempAlertStream;
  • Count pattern count pattern matches multiple events received under the same condition

    • define stream TemperatureStream (sensorID long, roomNo int,
          temp double);
      
      define stream RegulatorStream (deviceID long, roomNo int,
          tempSet double, isOn bool);
      
      @sink(type = 'log')
      define stream TemperatureDiffStream(roomNo int,
          tempDiff double);
      
      from every( e1 = RegulatorStream)
          -> e2 = TemperatureStream[e1.roomNo == roomNo] < 1: >
          -> e3 = RegulatorStream[e1.roomNo == roomNo]
      select e1.roomNo, e2[0].temp - e2[last].temp as tempDiff
      insert into TemperatureDiffStream;
  • Logical pattern matches the order and logical relationship of arrival time

    • define stream RegulatorStateChangeStream(deviceID long,
          roomNo int, tempSet double, action string);
      
      define stream RoomKeyStream(deviceID long, roomNo int,
          action string);
      
      @sink(type='log')
      define stream RegulatorActionStream(roomNo int, action string);
      
      from every e1=RegulatorStateChangeStream[ action == 'on' ]
           -> e2=RoomKeyStream
                  [ e1.roomNo == roomNo and action == 'removed' ]
              or e3=RegulatorStateChangeStream
                  [ e1.roomNo == roomNo and action == 'off']
      select e1.roomNo,
          ifThenElse( e2 is null, 'none', 'stop' ) as action
      having action != 'none'
      insert into RegulatorActionStream;
  • Non concurrent pattern detects the occurrence and expiration of events

    • define stream RegulatorStateChangeStream(deviceID long,
          roomNo int, tempSet double, action string);
      
      define stream TemperatureStream (roomNo int, temp double);
      
      @sink(type='log')
      define stream RoomTemperatureAlertStream(roomNo int);
      
      from e1=RegulatorStateChangeStream[action == 'on']
           -> not TemperatureStream[e1.roomNo == roomNo and
              temp <= e1.tempSet] for 30 sec
      select e1.roomNo as roomNo
      insert into RoomTemperatureAlertStream;
  • Simple sequence a simple sequence matches successive events that arrive in time

    • define stream StockRateStream (symbol string, price float,
          volume int);
      
      @sink(type='log')
      define stream PeakStockRateStream (symbol string,
          rateAtPeak float);
      
      partition with (symbol of StockRateStream)
      begin
      
          from every e1=StockRateStream,
              e2=StockRateStream[e1.price < price],
              e3=StockRateStream[e2.price > price]
              within 10 min
          select e1.symbol, e2.price as rateAtPeak
          insert into PeakStockRateStream ;
      end;
  • Sequence with count sorts the countable sequences by counting sequence

    • define stream TemperatureStream(roomNo int, temp double);
      
      @sink(type='log') 
      define stream PeekTemperatureStream(roomNo int,
          initialTemp double, peekTemp double, firstDropTemp double);
      
      partition with (roomNo of TemperatureStream)
      begin
          @info(name = 'temperature-trend-analyzer')
          from every e1=TemperatureStream,
               e2=TemperatureStream[ifThenElse(e2[last].temp is null,
                      e1.temp <= temp, e2[last].temp <= temp)]+,
               e3=TemperatureStream[e2[last].temp > temp]
          select e1.roomNo, e1.temp as initialTemp,
              e2[last].temp as peekTemp, e3.temp as firstDropTemp
      
          insert into PeekTemperatureStream ;
      end;
  • Logical sequence the logical sequence matches the sequence of repeated events and is logically sorted

    • define stream TempSensorStream(deviceID long,
          isActive bool);
      
      define stream HumidSensorStream(deviceID long,
          isActive bool);
      
      define stream RegulatorStream(deviceID long, isOn bool);
      
      @sink(type='log')
      define stream StateNotificationStream (deviceID long,
          tempSensorActive bool, humidSensorActive bool);
      
      from every e1=RegulatorStream[isOn == true],
          e2=TempSensorStream and e3=HumidSensorStream
      select e1.deviceID, e2.isActive as tempSensorActive,
          e3.isActive as humidSensorActive
      insert into StateNotificationStream;

service integration

  • HTTP Service Integration

    • @sink(type='http-call',
          publisher.url='http://localhost:8005/validate-loan',
          method='POST', sink.id='loan-validation',
          @map(type='json'))
      define stream LoanValidationStream (clientId long,
                          name string, amount double);
      --The producer broadcasts the data on HTTP call JSON
      @source(type='http-call-response', sink.id='loan-validation',
          http.status.code='2\d+',
          @map(type='json', @attributes(customerName='trp:name',
              clientId='trp:clientId', loanAmount='trp:amount',
              interestRate='validation-response.rate',
              totalYears='validation-response.years-approved')))
      define stream SuccessLoanRequestStream(clientId long,
                 customerName string, loanAmount double,
                 interestRate double, totalYears int);
      --The consumer HTTP call response only meets the condition that the status code starts with 2
      @source(type='http-call-response', sink.id='loan-validation',
          http.status.code='400',
          @map(type='json', @attributes(customerName='trp:name',
              clientId='trp:clientId',
              failureReason='validation-response.reason')))
      define stream FailureLoanRequestStream(clientId long,
                      customerName string, failureReason string);
      --The consumer HTTP call response only meets the condition that the status code is 400
      define stream LoanRequestStream (clientId long, name string,
                      amount double);
      
      @sink(type='log') 
      define stream LoanResponseStream(clientId long,
                      customerName string, message string);
      --Producer log
      @info(name = 'attribute-projection')
      from LoanRequestStream
      select clientId, name, amount
      insert into LoanValidationStream;
      
      @info(name = 'successful-message-generator')
      from SuccessLoanRequestStream
      select clientId, customerName,
          "Loan Request is accepted for processing" as message
      insert into LoanResponseStream;
      
      @info(name = 'failure-message-generator')
      from FailureLoanRequestStream
      select clientId, customerName,
              str:concat("Loan Request is rejected due to ",
                  failureReason) as message
      insert into LoanResponseStream;
    • Consumers whose HTTP response is normal and 400 are defined respectively. Through JSON verification, the results are broadcast in the form of log

  • gRPC service integration

    • define stream TicketBookingStream (name string, phoneNo string,
              movie string, ticketClass string, qty int,
              bookingTime long);
      
      @sink(type='grpc-call',
          publisher.url =
          'grpc://localhost:5003/org.wso2.grpc.EventService/process',
          sink.id= 'ticket-price', @map(type='json'))
      define stream TicketPriceFinderStream (name string,
              phoneNo string, movie string, ticketClass string,
              qty int, bookingTime long);
      --Producer
      @source(type='grpc-call-response',
          receiver.url =
          'grpc://localhost:9763/org.wso2.grpc.EventService/process',
          sink.id= 'ticket-price',
          @map(type='json', @attributes(customerName='trp:name',
              phoneNo='trp:phoneNo', movie='trp:movie',
              qty='trp:qty', bookingTime='trp:bookingTime',
              ticketPrice='price')))
      --Consumer
      define stream TicketPriceResponseStream (customerName string,
              phoneNo string, movie string, qty int,
              ticketPrice double, bookingTime long);
      
      @sink(type='log')
      define stream TotalTicketPaymentStream (customerName string,
              phoneNo string, movie string, totalAmount double,
              bookingTime long);
      --Log producer
      @info(name = 'filter-basic-ticket-bookings')
      from TicketBookingStream[ticketClass == "BASIC"]
      select name as customerName, phoneNo, movie,
          qty * 20.0 as totalAmount, bookingTime
      insert into TotalTicketPaymentStream;
      
      @info(name = 'filter-non-basic-tickets')
      from TicketBookingStream[ticketClass != "BASIC"]
      select *
      insert into TicketPriceFinderStream;
      
      @info(name = 'total-price-calculator')
      from TicketPriceResponseStream
      select customerName, phoneNo, movie,
          (qty * ticketPrice) as totalAmount, bookingTime
      insert into TotalTicketPaymentStream;

rate limiting

  • rate limit based on time

    • define stream APIRequestStream (apiName string, version string,
          tier string, user string, userEmail string);
      define stream UserNotificationStream (user string,
          apiName string, version string, tier string,
          userEmail string, throttledCount long);
      @info(name='api-throttler')
      from APIRequestStream#window.timeBatch(1 min, 0, true) 
      select apiName, version, user, tier, userEmail,
          count() as totalRequestCount
      group by apiName, version, user
      having totalRequestCount == 3 or totalRequestCount == 0
      insert all events into ThrottledStream;
      
      @info(name='throttle-flag-generator') 
      from ThrottledStream 
      select apiName, version, user, tier, userEmail,
          ifThenElse(totalRequestCount == 0, false, true)
              as isThrottled
      insert into ThrottleOutputStream;
      
      @info(name='notification-generator') 
      from ThrottleOutputStream[isThrottled]#window.time(1 hour) 
      select user, apiName, version, tier, userEmail,
          count() as throttledCount
      group by user, apiName, version, tier
      having throttledCount > 2
      output first every 15 min 
      insert into UserNotificationStream;

error handling

  • logging

    • define stream GlucoseReadingStream (locationRoom string,
          locationBed string, timeStamp string, sensorID long,
          patientFirstName string, patientLastName string,
          sensorValue double);
      
      @sink(type = 'http', on.error='log',
          publisher.url = "http://localhost:8080/logger",
          method = "POST",
          @map(type = 'json'))
      --Log error
      define stream AbnormalGlucoseReadingStream
          (timeStampInLong long, locationRoom string,
          locationBed string, sensorID long,
          patientFullName string, sensorReadingValue double);
      
      @info(name='abnormal-reading-identifier')
      from GlucoseReadingStream[sensorValue > 220]
      select math:parseLong(timeStamp) as timeStampInLong,
          locationRoom, locationBed, sensorID,
          str:concat(patientFirstName, " ", patientLastName)
              as patientFullName,
              sensorValue as sensorReadingValue
      insert into AbnormalGlucoseReadingStream;
  • wait&retry

    • define stream GlucoseReadingStream (locationRoom string,
          locationBed string, timeStamp string, sensorID long,
          patientFirstName string, patientLastName string,
          sensorValue double);
      
      @sink(type = 'http', on.error='wait',
          publisher.url = "http://localhost:8080/logger",
          method = "POST",
          @map(type = 'json'))
      --An error was encountered. Wait to exit and try again 
      -- on. Error configuration processing error resolution
      define stream AbnormalGlucoseReadingStream
          (timeStampInLong long, locationRoom string,
          locationBed string, sensorID long,
          patientFullName string, sensorReadingValue double);
      
      @info(name='abnormal-reading-identifier')
      from GlucoseReadingStream[sensorValue > 220]
      select math:parseLong(timeStamp) as timeStampInLong,
          locationRoom, locationBed, sensorID,
          str:concat(patientFirstName, " ", patientLastName)
              as patientFullName,
              sensorValue as sensorReadingValue
      insert into AbnormalGlucoseReadingStream;