Custom (JavaScript)

The Custom feed type allows the raw document to be processed by JavaScript to construct an appropriate tabular dataset.

Property Name
Description
Secured Variables Secured Variables resolve to values based on the User that is logged into the system. If you have configured Secured Variables in the system, you can include them here to filter the data that will be returned based on what User is accessing this Feed. For more information, see Configuring Secured Variables.
Script The JavaScript source used to process the raw input document (a string) into an appropriate tabular dataset (outputBundle). This is similar to the JavaScript Transform except the input is a string versus a tabular dataset (inputBundle).
Poll Interval (seconds)
This poll rate determines how often the server will check the data for updates.
Enable Server Subscription If set, the server will subscribe to the feed, just as a client widget would. This means that the data and any resources that would otherwise be allocated “on-demand” for the first user to view a Visualization that leverages the data produced by this feed are allocated when the server is started and maintained as long as this feed is configured.

Example: HTML Parsing

Prerequisites #

This example relies on a java library called jsoup: https://jsoup.org/download. The core library jar was installed in the <edge-home>/lib. The example is known to work with jsoup-1.10.3.jar.

URL for web data source: http://www.weather.gov/data/obhistory/KPWA.html.

Perform the following steps to complete this example:

  1. Download and save the jsoup JAR file (see above) into the directory: <edge-home>/lib
  2. Create a new Web Data connection with Destination =  http://w1.weather.gov/
  3. Create a Feed off of the Web Data Connection of “Custom” feed type:
    1. Start URI: /data/obhistory/KPWA.html
    2. Script: enter the script shown below
  4. Go to Data Preview step to view the parsed dataset.
Weather Custom Feed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
var jsoup = Java.type('org.jsoup.Jsoup');
load('classpath:static/lib/moment-with-locales.min.js');
function getAttributes(inputString, nodeVars, secVars) {
    var attributes = [];
    attributes.push({ name : 't', type : 'date', isId : true , units : 'millis' });
    attributes.push({ name : 'Wind' });
    attributes.push({ name : 'Visibility', type : 'number' });
    attributes.push({ name : 'Weather' });
    attributes.push({ name : 'Sky' });
    attributes.push({ name : 'Temperature', type : 'int' });
    attributes.push({ name : 'Dewpoint', type : 'int' });
    attributes.push({ name : 'Humidity', type : 'int' });
    attributes.push({ name : 'Rain', type : 'number' });
    return jsAttributesSuccess(attributes, "Example Success");
}
function getRecords(inputString, nodeVars, secVars) {
    var records = [];
    var doc = jsoup.parse(inputString);
    var rows = doc.select('table[cellspacing=3] tr[bgcolor="#eeeeee"],table[cellspacing=3] tr[bgcolor="#f5f5f5"]');
    rows.forEach(function(row) {
        var tds = row.select('td');
        var day = tds.get(0).text();
        var time = tds.get(1).text().split(':');
        var t = moment().startOf('day');
        while (t.date() != day) {
          t = t.subtract(1, 'days');
        }
        t.hour(time[0]);
        t.minute(time[1]);
        var txt = tds.get(15).text();
        var rain = (txt.length > 0) ? parseFloat(txt) : 0;
        var record = {
            't' : t.unix()*1000,
            'Wind' : tds.get(2).text(),
            'Visibility' : tds.get(3).text(),
            'Weather' : tds.get(4).text(),
            'Sky' : tds.get(5).text(),
            'Temperature' : parseInt(tds.get(6).text()),
            'Dewpoint' : parseInt(tds.get(7).text()),
            'Humidity' : parseInt(tds.get(10).text()),
            'Rain' : rain
        };
        records.push(record);
    });
    return jsRecordsSuccess(records, "Example Success");
}