Early work on background agent for search

agent.js

@@ -11,17 +11,20 @@ global.ROOTPATH = __dirname;
 // ----------------------------------------
 
 global.winston = require('winston');
-winston.info('[AGENT] Requarks Wiki BgAgent is initializing...');
+winston.info('[AGENT] Background Agent is initializing...');
 
 var appconfig = require('./models/config')('./config.yml');
 
-global.git = require('./models/git').init(appconfig, true);
+global.git = require('./models/git').init(appconfig);
 global.entries = require('./models/entries').init(appconfig);
 global.mark = require('./models/markdown');
+global.search = require('./models/search').init(appconfig);
 
 var _ = require('lodash');
 var moment = require('moment');
 var Promise = require('bluebird');
+var fs = Promise.promisifyAll(require("fs-extra"));
+var path = require('path');
 var cron = require('cron').CronJob;
 
 // ----------------------------------------
@@ -44,6 +47,7 @@ var job = new cron({
 		// Prepare async job collector
 
 		let jobs = [];
+		let repoPath = path.resolve(ROOTPATH, appconfig.datadir.repo);
 
 		// ----------------------------------------
 		// Compile Jobs
@@ -51,12 +55,58 @@ var job = new cron({
 
 		//-> Resync with Git remote
 
-		jobs.push(git.resync().then(() => {
+		jobs.push(git.onReady.then(() => {
+			return git.resync().then(() => {
 
-			//-> Purge outdated cache
+				//-> Stream all documents
 
-			return entries.purgeStaleCache();
+				let cacheJobs = [];
 
+				fs.walk(repoPath).on('data', function (item) {
+					if(path.extname(item.path) === '.md') {
+
+						let entryPath = entries.parsePath(entries.getEntryPathFromFullPath(item.path));
+						let cachePath = entries.getCachePath(entryPath);
+						
+						//-> Purge outdated cache
+
+						cacheJobs.push(
+							fs.statAsync(cachePath).then((st) => {
+								return moment(st.mtime).isBefore(item.stats.mtime) ? 'expired' : 'active';
+							}).catch((err) => {
+								return (err.code !== 'EEXIST') ? err : 'new';
+							}).then((fileStatus) => {
+
+								//-> Delete expired cache file
+
+								if(fileStatus === 'expired') {
+									return fs.unlinkAsync(cachePath).return(fileStatus);
+								}
+
+								return fileStatus;
+
+							}).then((fileStatus) => {
+
+								//-> Update search index
+
+								if(fileStatus !== 'active') {
+									return entries.fetchTextVersion(entryPath).then((content) => {
+										console.log(content);
+									});
+								}
+
+								return true;
+
+							})
+
+						);
+
+					}
+				});
+
+				return Promise.all(cacheJobs);
+
+			});
 		}));
 
 		// ----------------------------------------
@@ -73,7 +123,8 @@ var job = new cron({
 
 	},
 	start: true,
-	timeZone: 'UTC'
+	timeZone: 'UTC',
+	runOnInit: true
 });
 
 // ----------------------------------------