node-ff-assets/index.js

var NodeFsExtra    = require('fs-extra');
var NodeMinify     = require('minify');
var NodeEvents     = require('events');
var NodeFetch      = require('fetch');
var NodeFs         = require('fs');

var DEFAULT_DOWNLOAD_MAX_SIZE = (1024*1024) * 10; // 10mb
var DEFAULT_DOWNLOAD_TIMEOUT  = (1000*60)   * 1;  // 1min
var DEFAULT_OPTIONS = {
	downloadStartDelay: null,
	downloadForce: false,
	downloadOptions: {
		timeout: DEFAULT_DOWNLOAD_TIMEOUT,
		maxResponseLength: DEFAULT_DOWNLOAD_MAX_SIZE,
		agent: 'node-ff-assets asset fetcher'
	},
	
	linkTargetSingleResult: true,
	linkTarget: null,
	linkSources: [],
	linkMapping: [],
	
	assetHeader: '\n/* node-ff-assets: begin */\n\n',
	assetFooter: '\n/* node-ff-assets: end */\n\n',
	assetSeperator: '\n/* node-ff-assets: next */\n',
}

function ResourceBuilder(config, readFile) {
	this.config = config;
	this.startTime = 0;
	this.readFile = readFile || FunctionFactory.Constructor.readFile();
	NodeEvents.EventEmitter.call(this);
	
	this.run = function(callback) {
		configCheckSync(this.config);
		configDefaultsSync(this.config);
		runBuildSafe(this, this.readFile, callback);
	};
}

ResourceBuilder.prototype.__proto__ = NodeEvents.EventEmitter.prototype;

var FunctionFactory = {
	Constructor: {
		readFile: function() {
			return function(file,callback) {
				NodeFs.readFile(file, function(err, data) {
					if (err) {
						callback(err);
					} else {
						callback(null, data);
					}
				});
			};
		},
		readFileMinify: function(options) {
			if (options == null) {
				options = {};
			}
			return function(file, callback) {
				NodeMinify(file, options, function(err, data) {
					if (err) {
						callback(err);
					} else {
						callback(null, data);
					}
				});
			};
		},
	},
	Event: {
		log: {
			console: function(logType) {
				if (logType == null) {
					throw Error('no logType');
				}
				return function(logLevel, logMessage) {
					console.log(logLevel+': node-ff-assets-'+logType+' '+logMessage);
				};
			},
		},
		result: {
			objectFunction: function(object, functionName, resultKey) {
				if (object == null) {
					throw Error('no object');
				}
				if (functionName == null) {
					throw Error('no functionName');
				}
				return function(resultValue) {
					if (resultKey == null) {
						object[functionName](resultValue);
					} else {
						object[functionName](resultKey, resultValue);
					}
				};
			},
			objectAdd: function(object,resultKey) {
				return FunctionFactory.Event.result.objectFunction(object,'add',resultKey);
			},
			objectSet: function(object,resultKey) {
				return FunctionFactory.Event.result.objectFunction(object,'set',resultKey);
			},
			objectPut: function(object,resultKey) {
				return FunctionFactory.Event.result.objectFunction(object,'put',resultKey);
			},
			arrayPush: function(object,resultKey) {
				return FunctionFactory.Event.result.objectFunction(object,'push');
			},
		}
	}
}

function configCheckSync(config) {
	if (!config) {
		throw Error('no config');
	}
	if (!config.linkTarget) {
		throw Error('no config.linkTarget');
	}
	if (!config.linkSources) {
		throw Error('no config.linkSources');
	}
	if (!config.linkMapping) {
		throw Error('no config.linkMapping');
	}
}

function configDefaultsSync(config) {
	if (config.downloadOptions == null) {
		config.downloadOptions = DEFAULT_OPTIONS.downloadOptions;
	}
	if (config.assetHeader == null) {
		config.assetHeader = DEFAULT_OPTIONS.assetHeader;
	}
	if (config.assetFooter == null) {
		config.assetFooter = DEFAULT_OPTIONS.assetFooter;
	}
	if (config.assetSeperator == null) {
		config.assetSeperator = DEFAULT_OPTIONS.assetSeperator;
	}
}

function runBuildSafe(builder, readFile, callback) {
	runBuild(builder, readFile, function(err) {
		// send error
		if (err) {
			builder.emit('error',err);
		}
		// finish callback
		if (callback != null) {
			callback(err);
		}
	});	
}

function runBuild(builder, readFile, callback) {
	if (builder.config.downloadStartDelay) {
		setTimeout(function() {
			buildStart(builder, readFile, callback);
		}, builder.config.downloadStartDelay);
	} else {
		buildStart(builder, readFile, callback);
	}
}

function buildStart(builder, readFile, callback) {
	builder.startTime = new Date().getTime();
	builder.emit('begin');
	builder.emit('log','info','build begin for: '+builder.config.linkTarget);
	if (builder.config.downloadForce) {
		builder.emit('log','info','build using forced downloads.');
	}
	var targetFile = mapLocalFileSync(builder, builder.config.linkTarget);
	NodeFsExtra.ensureFile(targetFile, function (err) {
		if (err) {
			callback(err);
		} else {
			builder.emit('file-write-pre',targetFile);
			NodeFsExtra.writeFile(targetFile, builder.config.assetHeader, function(err) {
				if (err) {
					callback(err);
				} else {
					buildAsset(builder, targetFile, readFile, callback);
				}
			});
		}
	});
}

function buildAsset(builder, targetFile, readFile, callback) {
	var uriList = builder.config.linkSources;
	var localFileList = [];
	var resultUriList = [];
	var downloadList = [];
	
	for (i = 0; i < uriList.length; i++) {
		var assetItem = uriList[i];
		
		var remoteUrl = null;
		var remoteForce = null;
		var remoteIndex = assetItem.indexOf('@');
		if (remoteIndex > 0) {
			remoteUrl = assetItem.substring(remoteIndex + 1);
			if (remoteUrl.indexOf('@') == 0) {
				remoteUrl = assetItem.substring(remoteIndex + 2);
				remoteForce = true;
			}
			assetItem = assetItem.substring(0,remoteIndex);
		}
		var localFile = mapLocalFileSync(builder, assetItem);
		
		// override force on all files
		if (remoteForce==null && builder.config.downloadForce != null) {
			remoteForce = builder.config.downloadForce;
		}
		
		if (remoteUrl && (remoteForce || !NodeFs.existsSync(localFile))) {
			downloadList.push({
				remoteUrl: remoteUrl,
				localFile: localFile
			});
		}
		localFileList.push(localFile);
		resultUriList.push(assetItem);
	}
	downloadList.reverse();
	localFileList.reverse();
	
	downloadFileList(builder, downloadList, function(error) {
		if (error) {
			callback(err);
		} else {
			//for(....
			//if (!NodeFs.existsSync(localFile)) {
			//	Log.warn("illegal entry: "+localFile);
			//	continue;
			//}		
			aggregateFileList(builder, targetFile, localFileList, readFile, function(err) {
				if (err) {
					callback(err);
				} else {
					buildEnd(builder, targetFile, resultUriList, callback);
				}
			});
		}
	});
}

function mapLocalFileSync(builder, linkSource) {
	var uriMapping = builder.config.linkMapping;
	if (uriMapping) {
		uriMappingKeys = Object.keys(uriMapping);
		uriMappingKeys.sort(function(a, b) {
			return a.length < b.length; // longest first as we break on first hit
		});
		for (ii = 0; ii < uriMappingKeys.length; ii++) {
			var uriKey = uriMappingKeys[ii];
			var localPath = uriMapping[uriKey];
			var mapIndex = linkSource.indexOf(uriKey);
			if (mapIndex == 0) {
				return localPath+linkSource.substring(uriKey.length);
			}
		}
	}
	return linkSource;
}

function downloadFileList(builder, downloadList, callback) {
	if (downloadList.length == 0) {
		callback();
		return;
	}
	var download = downloadList.pop();
	downloadFile(builder, download.remoteUrl, download.localFile, function(err) {
		if (err) {
			callback(err);
		} else {
			downloadFileList(builder, downloadList, callback);
		}
	});
}

function downloadFile(builder, remoteUrl, localFile, callback) {
	builder.emit('file-download-pre',remoteUrl);
	builder.emit('log','debug','downloadFile: '+remoteUrl);
	NodeFsExtra.ensureFile(localFile, function(err) {
		if (err) {
			callback(err);
		} else {
			var stream = new NodeFetch.FetchStream(remoteUrl);
			stream.on('error', function(err) {
				callback(err);
			});
			stream.on('end', function() {
				builder.emit('file-download-post',remoteUrl);
				callback();
			});
			stream.pipe(NodeFs.createWriteStream(localFile));
		}
	});
}

function aggregateFileList(builder, targetFile, aggregateList, readFile, callback) {
	if (aggregateList.length == 0) {
		callback();
		return;
	}
	var aggregateFile = aggregateList.pop();
	builder.emit('file-read-pre',aggregateFile);
	readFile(aggregateFile, function(err, data) {
		if (err) {
			callback(err);
		} else {
			builder.emit('file-read-post',aggregateFile);
			builder.emit('log','debug','readFile: '+aggregateFile+' size: '+data.length);
			if (aggregateList.length > 0) {
				data = data + builder.config.assetSeperator;
			}
			NodeFs.appendFile(targetFile, data, function (err) {
				if (err) {
					callback(err);
				} else {
					aggregateFileList(builder, targetFile, aggregateList, readFile, callback);
				}
			});
		}
	});
}

function buildEnd(builder, targetFile, resultUriList, callback) {
	NodeFs.appendFile(targetFile, builder.config.assetFooter, function (err) {
		if (err) {
			callback(err);
		} else {
			builder.emit('file-write-post',targetFile);
			var buildTime = new Date().getTime() - builder.startTime;
			var buildResultSize = +resultUriList.length;
			
			if (builder.config.linkTargetSingleResult) {
				resultUriList = [builder.config.linkTarget];
				
			}
			var targetStats = NodeFs.statSync(targetFile);
			var targetSize = targetStats['size'];
			
			builder.emit('log','debug','target size: '+targetSize);
			builder.emit('log','info','build result size: '+resultUriList.length+' from: '+buildResultSize);
			builder.emit('log','info','build done in: '+buildTime+' ms.');
			builder.emit('result',resultUriList);
			builder.emit('end');
			callback();
		}
	});
}

function runBuilderList(builderList,callback) {
	if (callback == null) {
		callback = function(err) {
			if(err) {
				throw err; // mm
			}
		};
	}
	if (builderList.length == 0) {
		callback();
		return;
	}
	var builder = builderList.pop();
	builder.run(function(err) {
		if (err) {
			callback(err);
		} else {
			runBuilderList(builderList,callback);
		}
	});
}

function runAll(builderList,callback) {
	builderList.reverse();
	runBuilderList(builderList,callback);
}

function pushLinkSourcesSync(builder, linkPrefix, fileFolder) {
	NodeFs.readdirSync(fileFolder).forEach(function (file) {
		if (~file.indexOf('.js') || ~file.indexOf('.css')) {
			builder.linkSources.push(linkPrefix+file)
		}
	});
}

module.exports = {
	ResourceBuilder: ResourceBuilder,
	FunctionFactory: FunctionFactory,
	runAll: runAll,
	pushLinkSourcesSync: pushLinkSourcesSync,
}