Budapest HU
Sydney AU
London UK

GLEIF Walkthrough Example

Here we try to demonstrate using the microservices to compare records in a GLEIF XML file – which is fairly pointless!

Use the LEISMALL.XML file from the resources, below is an excerpt of it:

<?xml version='1.0' encoding='UTF-8'?>
<lei:LEIData xmlns:gleif="http://www.gleif.org/concatenated-file/header-extension/2.0" xmlns:lei="http://www.gleif.org/data/schema/leidata/2016">
<lei:LEIHeader>
<lei:ContentDate>2017-12-05T09:00:01+00:00</lei:ContentDate>
<lei:FileContent>GLEIF_FULL_PUBLISHED</lei:FileContent>
<lei:RecordCount>824170</lei:RecordCount>
 </lei:LEIHeader>
<lei:LEIRecords>
<lei:LEIRecord>
<lei:LEI>029200137F2K8AH5C573</lei:LEI>
<lei:Entity>
<lei:LegalName xml:lang="en">CAPITAL ASSETS LIMITED</lei:LegalName>
<lei:OtherEntityNames>
<lei:OtherEntityName xml:lang="fr" type="ALTERNATIVE_LANGUAGE_LEGAL_NAME">CAPITAL ASSETS LIMITED</lei:OtherEntityName>
</lei:OtherEntityNames>
<lei:LegalAddress xml:lang="en">
<lei:FirstAddressLine>9TH FLOOR BOOKSHOP HOUSE, 50/52 BROAD STREET</lei:FirstAddressLine>
<lei:AdditionalAddressLine>P.O. BOX 702 LAGOS</lei:AdditionalAddressLine>
<lei:City>LAGOS</lei:City>
<lei:Region>NG-LA</lei:Region>
<lei:Country>NG</lei:Country>
<lei:PostalCode>23401</lei:PostalCode>
</lei:LegalAddress>
<lei:HeadquartersAddress xml:lang="en">
<lei:FirstAddressLine>9TH FLOOR BOOKSHOP HOUSE, 50/52 BROAD STREET</lei:FirstAddressLine>
<lei:AdditionalAddressLine>P.O. BOX 702 LAGOS</lei:AdditionalAddressLine>
<lei:City>LAGOS</lei:City>
<lei:Region>NG-LA</lei:Region>
<lei:Country>NG</lei:Country>
</lei:HeadquartersAddress>
<lei:OtherAddresses>
<lei:OtherAddress xml:lang="fr" type="ALTERNATIVE_LANGUAGE_HEADQUARTERS_ADDRESS">
<lei:FirstAddressLine>9TH FLOOR BOOKSHOP HOUSE, 50/52 BROAD STREET</lei:FirstAddressLine>
<lei:AdditionalAddressLine>P.O. BOX 702 LAGOS</lei:AdditionalAddressLine>
<lei:City>LAGOS</lei:City>
<lei:Region>NG-LA</lei:Region>
<lei:Country>NG</lei:Country>
<lei:PostalCode>23401</lei:PostalCode>
</lei:OtherAddress>
</lei:OtherAddresses>
<lei:RegistrationAuthority>
<lei:RegistrationAuthorityID>RA000469</lei:RegistrationAuthorityID>
<lei:RegistrationAuthorityEntityID>RC286096</lei:RegistrationAuthorityEntityID>
</lei:RegistrationAuthority>
<lei:LegalJurisdiction>NG-LA</lei:LegalJurisdiction>
<lei:LegalForm>
<lei:EntityLegalFormCode>8888</lei:EntityLegalFormCode>
<lei:OtherLegalForm>LIMITED</lei:OtherLegalForm>
</lei:LegalForm>
<lei:EntityStatus>ACTIVE</lei:EntityStatus>
</lei:Entity>
<lei:Registration>
<lei:InitialRegistrationDate>2014-11-06T00:00:00Z</lei:InitialRegistrationDate>
<lei:LastUpdateDate>2017-11-07T09:40:44+08:00</lei:LastUpdateDate>
<lei:RegistrationStatus>LAPSED</lei:RegistrationStatus>
<lei:NextRenewalDate>2017-11-06T00:00:00Z</lei:NextRenewalDate>
<lei:ManagingLOU>029200067A7K6CH0H586</lei:ManagingLOU>
<lei:ValidationSources>FULLY_CORROBORATED</lei:ValidationSources>
<lei:ValidationAuthority>
<lei:ValidationAuthorityID>RA000469</lei:ValidationAuthorityID>
</lei:ValidationAuthority>
</lei:Registration>
</lei:LEIRecord>
</lei:LEIRecords>
</lei:LEIData>

Using CURL and MARS we can convert it to JSON:

curl --data @leismall.xml -H "Content-Type: application/xml" -X POST http://localhost:8080/XMLtoJSON/GLEIF/GLEIF/LEIRecord

Output will appear in JSON:

[{
"Entity": {
"LegalForm": {
"OtherLegalForm": "LIMITED",
"EntityLegalFormCode": "8888"
},
"RegistrationAuthority": {
"RegistrationAuthorityEntityID": "RC286096",
"RegistrationAuthorityID": "RA000469"
},
"OtherEntityNames": [],
"LegalAddress": {
"AdditionalAddressLine": "P.O. BOX 702 LAGOS",
"PostalCode": "23401",
"FirstAddressLine": "9TH FLOOR BOOKSHOP HOUSE, 50/52 BROAD STREET",
"City": "LAGOS",
"Country": "NG",
"Region": "NG-LA"
},
"OtherAddresses": [
{
"OtherAddress": {
"Country": "NG",
"Region": "NG-LA",
"City": "LAGOS",
"AdditionalAddressLine": "P.O. BOX 702 LAGOS",
"PostalCode": "23401",
"FirstAddressLine": "9TH FLOOR BOOKSHOP HOUSE, 50/52 BROAD STREET"
}
}
],
"HeadquartersAddress": {
"City": "LAGOS",
"Region": "NG-LA",
"FirstAddressLine": "9TH FLOOR BOOKSHOP HOUSE, 50/52 BROAD STREET",
"AdditionalAddressLine": "P.O. BOX 702 LAGOS",
"Country": "NG"
},
"LegalName": "CAPITAL ASSETS LIMITED",
"EntityStatus": "ACTIVE",
"LegalJurisdiction": "NG-LA"
},
"LEI": "029200137F2K8AH5C573",
"Registration": {
"ValidationAuthority": {
"ValidationAuthorityID": "RA000469"
},
"InitialRegistrationDate": "2014-11-06T00:00:00Z",
"ManagingLOU": "029200067A7K6CH0H586",
"LastUpdateDate": "2017-11-07T09:40:44+08:00",
"ValidationSources": "FULLY_CORROBORATED",
"NextRenewalDate": "2017-11-06T00:00:00Z",
"RegistrationStatus": "LAPSED"
}
}
]

If you capture the output of the above command, you can use the Json Objects in the other MARS methods like compare, index or standardise:

curl --data @leismall.xml -H "Content-Type: application/xml" -X POST http://localhost:8080/XMLtoJSON/GLEIF/GLEIF/LEIRecord | curl --data @- -H "Content-Type: application/json" -X POST http://localhost:8080/compare/GLEIF/GLEIF

Although I really don’t recommend you do this!! It will compare all the records in the file with themselves – which could take some time. Alternatively the output of the XMLtoJSON should be passed to the index method and only the records that share the same keys produced from this should be compared. Using NodeJS and CURL here is an example:

COMPARISON Example using CURL and NODEJS

  1. Convert the XML file to a json file
curl --data @leismall.xml -H "Content-Type: application/xml" -X POST http://localhost:8080/XMLtoJSON/GLEIF/GLEIF/LEIRecord > ./objects.json

2.  Use the output into a series of calls to get the proposed index keys for the objects.

var fs = require('fs');
var http = require('http');
var array = require("./objects.json");
var closed=0;
var arrayData=new Array();
for (var i=0; i<array.length; i++){
  const options = {
   hostname: 'localhost',
   port: 8080,
   path: '/index/GLEIF/GLEIF/?isMatch=true&isSearch=false&includeOrig=true',
   method: 'POST',
   headers: {
   'Content-Type': 'application/json',
   'Content-Length': Buffer.byteLength(JSON.stringify(array[i]))
  }
 };
 const req = http.request(options, (res) => {
 var stringData="";
 res.setEncoding('utf8');
 res.on('data', (chunk) => {
   if (!res.stringData)
    res.stringData=chunk
   else
    res.stringData+=chunk;
   });
  res.on('end', () => {
   req.end();
   closed++;
   var arrayItems=JSON.parse(res.stringData);
   for (var j=0; j<arrayItems.length; j++){
     arrayData.push(arrayItems[j]);
   }
   if (closed==array.length){
    var stream = fs.createWriteStream("index.json");
    stream.write("[");
    for (var k=0; k<arrayData.length; k++){
    if (k!=0) 
     stream.write(",\n");
     stream.write(JSON.stringify(arrayData[k]));
    }
    stream.write("]");
    stream.close();
   }
  });
 });

 req.on('error', (e) => {
   console.error(`problem with request: ${e.message}`);
 });

 req.write(JSON.stringify(array[i]));
}

Save this as index.js and this is then executed using nodejs:

> node index.js

Note the use of the includeOrig=true, this spurts out the original record to the file, it is inefficient and you should not use this unless there is no other way to store the original records indexed by their keys, in this example the LEI is the key for the GLEIF record and will always be returned by the index method.

And example output from the index will look like this:

...{"LEI":"029200417A8N7XH0I647","Table":"GLEIF","System":"GLEIF","key":"RADSTARNG","isSearch":false,"isMatch":true,"isExactIndex":false,"indexName":"CompanyNameIdx2","purposeName":"Company_Index","origRecord":{"Entity":{"LegalForm":{"OtherLegalForm":"LIMITED","EntityLegalFormCode":"8888"},"RegistrationAuthority":{"RegistrationAuthorityEntityID":"RC200303","RegistrationAuthorityID":"RA000469"},"OtherEntityNames":[],"LegalAddress":{"AdditionalAddressLine":"LAGOS","PostalCode":"23401","FirstAddressLine":"70 INTERNATIONAL AIRPORT ROAD","City":"LAGOS","Country":"NG","Region":"NG-LA"},"OtherAddresses":[{"OtherAddress":{"Country":"NG","Region":"NG-LA","City":"LAGOS","AdditionalAddressLine":"LAGOS","PostalCode":"23401","FirstAddressLine":"70 INTERNATIONAL AIRPORT ROAD"}}],"HeadquartersAddress":{"City":"LAGOS","Region":"NG-LA","FirstAddressLine":"70 INTERNATIONAL AIRPORT ROAD","AdditionalAddressLine":"LAGOS","Country":"NG"},"LegalName":"RED STAR EXPRESS PLC","EntityStatus":"ACTIVE","LegalJurisdiction":"NG-LA"},"LEI":"029200417A8N7XH0I647","Registration":{"ValidationAuthority":{"ValidationAuthorityID":"RA000469"},"InitialRegistrationDate":"2015-03-05T00:00:00Z","ManagingLOU":"029200067A7K6CH0H586","LastUpdateDate":"2017-03-27T16:00:00Z","ValidationSources":"FULLY_CORROBORATED","NextRenewalDate":"2018-03-05T00:00:00Z","RegistrationStatus":"ISSUED"}}}
...

The key and origRecord are the only items we care about.

3.  Group the records by the keys returned, and score the groups:

var http = require('http');
var array = require("./index.json");
var map={};
var keylist=new Array();
//group records by keys produced
for (var i=0; i<array.length; i++){
  var keyobject=array[i];
  var key=keyobject.key;  
  var object=keyobject.origRecord;
  if (!map[key]){
    map[key]=new Array();
    keylist.push(key);
  }
  map[key].push(object);
}
array.length=0; //save some memory
//call the comparison for the grouped records
var closed=0;
var arrayData=new Array();
for (var m=0; m<keylist.length; m++){
  array=map[keylist[m]];
  const options = {
    hostname: 'localhost',
    port: 8080,
    path: '/compare/GLEIF/GLEIF/',
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Content-Length': Buffer.byteLength(JSON.stringify(array))
    }
  };
  const req = http.request(options, (res) => {
  var stringData="";
  res.setEncoding('utf8');
  res.on('data', (chunk) => {
    if (!res.stringData)
      res.stringData=chunk
    else
      res.stringData+=chunk;
  });
  res.on('end', () => {
    req.end();
    closed++;
    var arrayItems=JSON.parse(res.stringData);
    for (var j=0; j<arrayItems.length; j++){
      arrayData.push(arrayItems[j]);
    }
    if (closed==keylist.length){
      var stream = fs.createWriteStream("compare.json");
      stream.write("[");
      for (var k=0; k<arrayData.length; k++){
        if (k!=0)
          stream.write(",\n");
        stream.write(JSON.stringify(arrayData[k]));
      }
      stream.write("]");
      stream.close();
    }
  });
});

req.on('error', (e) => {
console.error(`problem with request: ${e.message}`);
});

// write data to request body
req.write(JSON.stringify(array));

Save this as compare.js and this is then executed using nodejs:

> node compare.js

This will produce an output file with records in the following form, all valid comparisons will be included regardless of their score – the important thing to note is the acceptance for the match in the output – the values for this would be Auto-Match, Reject, or User-Review.

{
"score": 100,
"acceptance": "Auto-Match",
"action": "EID",
"actionText": "Link",
"rule": {
"canMatchSameSystem": true,
"systemMatchType": 0,
"lowScore": 80,
"highScore": 95,
"actionText": ""
},
"records": [{
"Entity": {
"LegalForm": {
"OtherLegalForm": "LIMITED",
"EntityLegalFormCode": "8888"
},
"RegistrationAuthority": {
"RegistrationAuthorityEntityID": "RC50599",
"RegistrationAuthorityID": "RA000469"
},
"OtherEntityNames": [],
"LegalAddress": {
"AdditionalAddressLine": "OTOLO NNEWI ANAMBRA STATE",
"PostalCode": "23401",
"FirstAddressLine": "17 OSITA ONYEJIANYA STREET ANUKA",
"City": "NNEWI",
"Country": "NG",
"Region": "NG-AN"
},
"OtherAddresses": [{
"OtherAddress": {
"Country": "NG",
"Region": "NG-AN",
"City": "NNEWI",
"AdditionalAddressLine": "OTOLO NNEWI ANAMBRA STATE",
"PostalCode": "23401",
"FirstAddressLine": "17 OSITA ONYEJIANYA STREET ANUKA"
}
}],
"HeadquartersAddress": {
"City": "NNEWI",
"Region": "NG-AN",
"FirstAddressLine": "17 OSITA ONYEJIANYA STREET ANUKA",
"AdditionalAddressLine": "OTOLO NNEWI ANAMBRA STATE",
"Country": "NG"
},
"LegalName": "CUTIX PLC",
"EntityStatus": "ACTIVE",
"LegalJurisdiction": "NG-AN"
},
"LEI": "029200894A3T8E0D9I44",
"Registration": {
"ValidationAuthority": {
"ValidationAuthorityID": "RA000469"
},
"InitialRegistrationDate": "2015-11-23T00:00:00Z",
"ManagingLOU": "029200067A7K6CH0H586",
"LastUpdateDate": "2017-11-22T23:00:06Z",
"ValidationSources": "FULLY_CORROBORATED",
"NextRenewalDate": "2018-11-23T00:00:00Z",
"RegistrationStatus": "ISSUED"
},
"standardized": {
"Company_Name": [{
"Class": "com.identiza.entity.resolve.types.StandardCompanyName",
"gradient": 0,
"baseTokens": ["CUTIX", "PLC"],
"originalWords": ["CUTIX", "PLC"]
}],
"RegCoNum": [{
"Class": "com.identiza.entity.resolve.types.StandardString",
"baseTokens": ["RC50599"],
"originalWords": ["RC50599"]
}]
}
}, {
"Entity": {
"LegalForm": {
"OtherLegalForm": "LIMITED",
"EntityLegalFormCode": "8888"
},
"RegistrationAuthority": {
"RegistrationAuthorityEntityID": "RC50599",
"RegistrationAuthorityID": "RA000469"
},
"OtherEntityNames": [],
"LegalAddress": {
"AdditionalAddressLine": "OTOLO NNEWI ANAMBRA STATE",
"PostalCode": "23401",
"FirstAddressLine": "17 OSITA ONYEJIANYA STREET ANUKA",
"City": "NNEWI",
"Country": "NG",
"Region": "NG-AN"
},
"OtherAddresses": [{
"OtherAddress": {
"Country": "NG",
"Region": "NG-AN",
"City": "NNEWI",
"AdditionalAddressLine": "OTOLO NNEWI ANAMBRA STATE",
"PostalCode": "23401",
"FirstAddressLine": "17 OSITA ONYEJIANYA STREET ANUKA"
}
}],
"HeadquartersAddress": {
"City": "NNEWI",
"Region": "NG-AN",
"FirstAddressLine": "17 OSITA ONYEJIANYA STREET ANUKA",
"AdditionalAddressLine": "OTOLO NNEWI ANAMBRA STATE",
"Country": "NG"
},
"SuccessorEntity": [],
"LegalName": "CUTIX PLC",
"EntityStatus": "INACTIVE",
"LegalJurisdiction": "NG-AN"
},
"LEI": "029200894A3T8E0D9I41",
"Registration": {
"ValidationAuthority": {
"ValidationAuthorityID": "RA000469"
},
"InitialRegistrationDate": "2015-11-23T00:00:00Z",
"ManagingLOU": "029200067A7K6CH0H586",
"LastUpdateDate": "2017-03-27T16:00:00Z",
"ValidationSources": "FULLY_CORROBORATED",
"NextRenewalDate": "2016-11-23T00:00:00Z",
"RegistrationStatus": "ANNULLED"
},
"standardized": {
"Company_Name": [{
"Class": "com.identiza.entity.resolve.types.StandardCompanyName",
"gradient": 0,
"baseTokens": ["CUTIX", "PLC"],
"originalWords": ["CUTIX", "PLC"]
}],
"RegCoNum": [{
"Class": "com.identiza.entity.resolve.types.StandardString",
"baseTokens": ["RC50599"],
"originalWords": ["RC50599"]
}]
}
}]
}


Resources you may need:

GLEIF Sample (XML)

GLEIF Schema