fix: use latest dataset for building entity files

Author: patrickhulce

bin/build-entity-json-files.js

@@ -2,7 +2,10 @@ const _ = require('lodash')
 const fs = require('fs')
 const path = require('path')
 const JSON5 = require('json5')
-const {importMergedData} = require('./shared/merge-entity-origin-data')
+const {
+  importMergedData,
+  getEntityDatasetsMostRecentFirst,
+} = require('./shared/merge-entity-origin-data')
 
 const DIST_DIR = path.join(__dirname, '../dist')
 const DATA_DIR = path.join(__dirname, '../data')
@@ -16,7 +19,7 @@ function cleanStatsFromEntity(entity) {
 const sourceEntities = JSON5.parse(fs.readFileSync(`${DATA_DIR}/entities.json5`, 'utf8'))
 fs.writeFileSync(`${DIST_DIR}/entities.json`, JSON.stringify(sourceEntities))
 
-const httpArchiveData = importMergedData('2019-04-01-entity-scripting.json')
+const httpArchiveData = importMergedData(getEntityDatasetsMostRecentFirst()[0])
 const {getEntity} = require('../lib/index.js') // IMPORTANT: require this after entities have been written
 const entityExecutionStats = _(httpArchiveData)
   .groupBy(({domain}) => {

bin/shared/merge-entity-origin-data.js

@@ -14,6 +14,14 @@ function importDataset(filename) {
 }
 
 module.exports = {
+  importDataset,
+  getEntityDatasetsMostRecentFirst() {
+    return fs
+      .readdirSync(DATA_FOLDER)
+      .filter(f => f.includes('entity-scripting'))
+      .sort()
+      .reverse()
+  },
   importMergedData(entityFilename) {
     const originFilename = entityFilename.replace('entity-scripting', 'origin-scripting')
 

lib/index.test.js

@@ -36,7 +36,7 @@ describe('getEntity', () => {
   it('works for direct domain usage', () => {
     expect(getEntity('https://js.connect.facebook.net/lib.js')).toMatchInlineSnapshot(`
 Object {
-  "averageExecutionTime": 142.24502022559832,
+  "averageExecutionTime": 161.0291502603836,
   "categories": Array [
     "social",
   ],
@@ -60,16 +60,16 @@ Object {
   ],
   "homepage": "https://www.facebook.com",
   "name": "Facebook",
-  "totalExecutionTime": 288666212,
-  "totalOccurrences": 2029359,
+  "totalExecutionTime": 578108820,
+  "totalOccurrences": 3590088,
 }
 `)
   })
 
   it('works for inferred domain usage', () => {
     expect(getEntity('https://unknown.typekit.net/fonts.css')).toMatchInlineSnapshot(`
 Object {
-  "averageExecutionTime": 91.44604519774012,
+  "averageExecutionTime": 93.6581576026637,
   "categories": Array [
     "cdn",
   ],
@@ -84,8 +84,8 @@ Object {
   ],
   "homepage": "https://fonts.adobe.com/",
   "name": "Adobe TypeKit",
-  "totalExecutionTime": 971157,
-  "totalOccurrences": 10620,
+  "totalExecutionTime": 1940878,
+  "totalOccurrences": 20723,
 }
 `)
   })