initial commit

.gitignore

@@ -0,0 +1,53 @@
+#
+# Git ignore for fsim
+#
+
+### Forbidden
+*.log
+*.cache
+*.class
+*.so
+*.dll
+
+### Temp
+*~
+.tmp
+*.bak
+*.swp
+.~lock.*
+.kate-swp
+
+### Eclipse
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+
+### VS Code
+.vscode/
+
+# mac finder
+.DS_Store
+
+# windows exploder
+Thumbs.db
+
+# kde dolphin
+.directory

README.md

@@ -1,2 +1,4 @@
 # fsim
 
+Old ftp search engine
+

WebRoot/META-INF/MANIFEST.MF

@@ -0,0 +1,3 @@
+Manifest-Version: 1.0
+Class-Path: 
+

WebRoot/META-INF/context.xml

@@ -0,0 +1,74 @@
+<?xml version="1.0"?>
+<Context 		path="/" 
+					docBase="${catalina.home}/webapps/ROOT" 
+					debug="5" 
+					reloadable="true" 
+					crossContext="true" 
+				>
+
+
+	<Resource	name="mail/Session"
+				auth="Container"
+				type="javax.mail.Session"
+			/>
+	<ResourceParams name="mail/Session">
+		<parameter>
+			<name>mail.smtp.host</name>
+			<value>localhost</value>
+		</parameter>
+	</ResourceParams>
+
+<!-- 
+	 <Resource	name="jdbc/fsim_dba"
+ 	 			description="DB Connection for fsim"
+				scope="Shareable"
+				auth="Container"
+				type="com.mchange.v2.c3p0.ComboPooledDataSource"
+	 					
+	 			maxPoolSize="4"
+	 			minPoolSize="2"
+	 			acquireIncrement="1"
+	 					
+	 			driverClass="org.postgresql.Driver"
+	 			user="fsim"
+	 			password="fsim"
+	 			factory="org.apache.naming.factory.BeanFactory"
+	 			jdbcUrl="jdbc:postgresql://localhost/fsim"
+	 		/>
+	 	 -->
+        <Resource       name="jdbc/fsim_dba"
+                        scope="Shareable"
+                        type="javax.sql.DataSource"
+                        auth="Container"
+                        factory="org.apache.tomcat.dbcp.dbcp.BasicDataSourceFactory"
+                        accessToUnderlyingConnectionAllowed="false"
+
+                        defaultAutoCommit="false"
+                        defaultReadOnly="false"
+                        defaultTransactionIsolation="READ_COMMITTED"
+
+                        validationQuery="select 1"
+                        testOnBorrow="true"
+                        testOnReturn="false"
+                        testWhileIdle="false"
+
+                        initialSize="5"
+                        minIdle="2"
+                        maxIdle="10"
+                        maxActive="20"
+                        maxWait="6000"
+
+                        removeAbandoned="true"
+                        removeAbandonedTimeout="5"
+                        logAbandoned="true"
+
+                        timeBetweenEvictionRunsMillis="2000"
+                        minEvictableIdleTimeMillis="20000"
+                        numTestsPerEvictionRun="5"
+
+                        driverClassName="org.postgresql.Driver"
+                        username="fsim" password="fsim"
+                        url="jdbc:postgresql://localhost/fsim"
+        />
+
+</Context>

WebRoot/WEB-INF/classes/com/idca/fsim/resources/i18n/fsim_en_US.properties

@@ -0,0 +1,36 @@
+
+fsim.title				= FSIM: File Search Index Machine
+
+fsim.logo.alt			= fsim-logo
+fsim.logo.title			= FSIM Searcher
+
+fsim.header.title		= File Search
+
+fsim.topbar.results		= Results:
+fsim.topbar.resultsin 	= found in:
+fsim.topbar.inms		= ms.
+
+fsim.search.button		= Search
+
+
+fsim.bottombar.rendered = Rendered in:
+fsim.bottombar.inms		= ms.
+
+fsim.copyright			= <br/>Fsim is a product of idca.nl<br/>(c)2005 idca
+
+fsim.tech.vcss.alt		= Valid CSS
+fsim.tech.vcss.title	= This page has an valid css file.
+fsim.tech.xhtml11.alt	= Valid XHML 1.1
+fsim.tech.xhtml11.title	= This page is build with XHTML technolocy.
+fsim.tech.debian.alt	= debian-logo
+fsim.tech.debian.title	= This service runs on Debian.
+fsim.tech.java.alt		= java-logo
+fsim.tech.java.title	= This service is written in java.
+fsim.tech.postgres.alt	= postgresql-logo
+fsim.tech.postgres.title= This service uses an PostgreSQL DB.
+fsim.tech.linux.alt		= linux-logo
+fsim.tech.linux.title	= The OS runs Linux.
+fsim.tech.tomcat.alt	= tomcat-logo
+fsim.tech.tomcat.title	= Tomcat 5 J2EE WebServer.
+
+fsim.text				= foo {0} bar

WebRoot/WEB-INF/classes/com/idca/fsim/resources/i18n/fsim_nl.properties

@@ -0,0 +1,9 @@
+
+
+fsim.test		= Mmmm dit is een test
+
+
+fsim.copyright	= Fsim is a product of idca.nl<br/>(c)2005 idca
+
+
+

WebRoot/WEB-INF/classes/com/idca/fsim/resources/logging/.cvsignore

@@ -0,0 +1 @@
+logging-development.properties

WebRoot/WEB-INF/classes/com/idca/fsim/resources/logging/logging-development-example.properties

@@ -0,0 +1,26 @@
+#
+# COPY THIS FILE TO: logging-development.properties
+# and set the properties correctly
+#
+
+# Specify the handlers to create in the root logger
+# (all loggers are children of the root logger)
+# The following creates two handlers
+handlers = java.util.logging.ConsoleHandler
+
+# Set the default logging level for new ConsoleHandler instances
+java.util.logging.ConsoleHandler.level = ALL
+
+# Set the default formatter for new ConsoleHandler instances  ( was java.util.logging.SimpleFormatter)
+java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
+    
+# Set the default logging level for the root logger
+.level = INFO
+
+#org.apache.tomcat.level = ALL
+#org.apache.catalina.level = ALL
+#org.apache.jasper.level = ALL
+#org.apache.commons.level = ALL
+#org.apache.myfaces.level = ALL
+#org.hibernate.level = ALL
+#com.opensymphony.oscache.level = ALL

WebRoot/WEB-INF/classes/com/idca/fsim/resources/logging/logging.properties

@@ -0,0 +1,29 @@
+#
+# THIS ARE THE LIVE logging properties
+# when the context parameter "DEVELOPMENT" on true is set
+# then we will try to read logging-development.properties.
+# this parameter should be set in web-override.xml
+#
+
+#
+# Specify the handlers to create in the root logger
+# (all loggers are children of the root logger)
+# The following creates two handlers
+handlers = java.util.logging.ConsoleHandler
+
+# Set the default logging level for new ConsoleHandler instances
+java.util.logging.ConsoleHandler.level = INFO
+
+# Set the default formatter for new ConsoleHandler instances  ( was java.util.logging.SimpleFormatter)
+java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
+    
+# Set the default logging level for the root logger
+.level = INFO
+
+#org.apache.tomcat.level = ALL
+#org.apache.catalina.level = ALL
+#org.apache.jasper.level = ALL
+#org.apache.commons.level = ALL
+#org.apache.myfaces.level = ALL
+#org.hibernate.level = ALL
+#com.opensymphony.oscache.level = ALL

WebRoot/WEB-INF/fsim.xml

@@ -0,0 +1,92 @@
+<?xml version="1.0"?>
+<fsimConfig name="WTH-SEARCH">
+	
+	<pluginPage	name="file_mirrors"
+				file="/fragments/searchers/files/file_mirrors.jsp"
+		/>
+	<pluginPage	name="file_redirect"
+				file="/fragments/searchers/files/file_redirect.jsp"
+		/>
+	<pluginPage	name="host_browse"
+				file="/fragments/searchers/files/host_browse.jsp"
+		/>
+	<pluginPage	name="host_redirect"
+				file="/fragments/searchers/files/host_redirect.jsp"
+		/>
+	<pluginPage	name="path_browse"
+				file="/fragments/searchers/files/path_browse.jsp"
+		/>
+	<pluginPage	name="path_redirect"
+				file="/fragments/searchers/files/path_redirect.jsp"
+		/>		
+	
+	<!-- 
+	<database	name="fsim_db"
+				driver="org.postgresql.Driver"
+				url="jdbc:postgresql://dbhost/fsim"
+				username="fsimdb"
+				password="bdmisf"
+		/>
+	-->
+		
+	<listener class="com.idca.fsim.searchers.files.FilesListener"/>
+	
+	<optionRenderer	dataModel="com.idca.fsim.searchers.files.models.File"
+					optionRenderer="com.idca.fsim.searchers.files.renderers.options.FileDownloadOptionRenderer"
+			/>
+	<optionRenderer	dataModel="com.idca.fsim.searchers.files.models.File"
+					optionRenderer="com.idca.fsim.searchers.files.renderers.options.FileDownloadOptionRenderer"
+			/>
+	<optionRenderer	dataModel="com.idca.fsim.searchers.files.models.File"
+					optionRenderer="com.idca.fsim.searchers.files.renderers.options.FileDownloadOptionRenderer"
+			/>
+
+	<!--  this is really an DB saerchSource...maybe include others.(v2) -->
+	<searchSource	name="fsim-files"
+					searchSource="com.idca.fsim.core.plugins.DBSearchSource"
+					dataModel="com.idca.fsim.searchers.generic.models.File"
+					indexer="com.idca.fsim.core.plugins.DBIndexer"
+					resultCollector="com.idca.fsim.searchers.files.collectors.FileResultCollector"
+					resultRenderer="com.idca.fsim.searchers.files.renderers.FileResultRenderer"
+					resultCacheResolver=""
+					
+					dbName="fsim_dba"
+					tableName="file"
+					idColumnName="id"
+					searchColumnName="name"
+		>
+		<!-- 
+		<initParam	name="DB_NAME" value="fsim_dba"/>
+		<initParam	name="TABLE_NAME" value="fsim_file"/>
+		<initParam	name="ID_COLUMN_NAME" value="file"/>
+		<initParam	name="SEARCH_COLUMN_NAME" value="name"/>
+		
+		<hitParamLong1 name="size"/>
+		<hitParamLong2 name="date"/>
+		-->
+	</searchSource>
+
+	<searchSource	name="fsim-paths"
+					searchSource="com.idca.fsim.core.plugins.DBSearchSource"
+					dataModel="com.idca.fsim.searchers.generic.models.Path"
+					indexer="com.idca.fsim.core.plugins.DBIndexer"
+					dbName="fsim_dba"
+					tableName="path"
+					idColumnName="id"
+					searchColumnName="directory"
+					resultCollector="com.idca.fsim.searchers.files.collectors.PathResultCollector"
+					resultRenderer="com.idca.fsim.searchers.files.renderers.PathResultRenderer"
+		/>
+	<searchSource	name="fsim-hosts"
+					searchSource="com.idca.fsim.core.plugins.DBSearchSource"
+					dataModel="com.idca.fsim.searchers.generic.models.Host"
+					indexer="com.idca.fsim.core.plugins.DBIndexer"
+					dbName="fsim_dba"
+					tableName="host"
+					idColumnName="id"
+					searchColumnName="hostname"
+					resultCollector="com.idca.fsim.searchers.files.collectors.HostResultCollector"
+					resultRenderer="com.idca.fsim.searchers.files.renderers.HostResultRenderer"
+		/>
+
+</fsimConfig>

WebRoot/WEB-INF/web.xml

@@ -0,0 +1,175 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<web-app version="2.4" 
+	xmlns="http://java.sun.com/xml/ns/j2ee" 
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
+	xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee 
+	http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd">
+<!-- 
+	Copyright 2004-2006 IDCA. All rights reserved.
+	IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+-->
+
+	<!-- 
+		=========== GENERAL CONFIG
+	-->
+
+	<display-name>FSIM</display-name>
+	<description>FSIM: File Search Index Machine</description>
+
+	<session-config>
+		<session-timeout>1</session-timeout>
+		<!--  An 1min session. -->
+	</session-config>
+	
+	<welcome-file-list>
+		<!--  this file redirects to /search -->
+		<welcome-file>/index.jsp</welcome-file>
+	</welcome-file-list>
+	
+	<context-param>
+		<description>The fsimcontext xml file</description>
+		<param-name>FSIM_CONFIG_FILE</param-name>
+		<param-value>/WEB-INF/fsim.xml</param-value>
+	</context-param>
+
+	<context-param>	
+		<description>
+			All servlets / listeners and filters use
+			this context.
+		</description>
+		<param-name>FsimContextName</param-name>
+		<param-value>WTH-SEARCH</param-value>
+	</context-param>
+	
+	<context-param>	
+		<description>
+			The page when a host is blocked.
+			note: should be on other server ivm DDOS
+		</description>
+		<param-name>hostBlockedRedirectURL</param-name>
+		<param-value>/denied/host_blocked.jsp</param-value>
+	</context-param>
+
+	
+	<!-- 
+		================== LISTENERS ========================
+	-->
+
+
+	<listener>
+		<listener-class>com.idca.fsim.core.FsimListener</listener-class>
+		<!--  Start fsim and loads config. -->
+	</listener>
+	<listener>
+		<listener-class>com.idca.fsim.core.indexer.IndexerManagerListener</listener-class>
+		<!--  Start fsim and loads config. -->
+	</listener>
+
+
+	<!-- 
+		======================== SERVLETS ======================
+	-->
+
+
+	<servlet>
+		<description>Fsim Search Servlet</description>
+		<display-name>Fsim Search Servlet</display-name>
+		<servlet-name>searchServlet</servlet-name>
+		<servlet-class>com.idca.fsim.web.SearchServlet</servlet-class>
+	</servlet>
+	<servlet-mapping>
+		<servlet-name>searchServlet</servlet-name>
+		<url-pattern>/search</url-pattern>
+	</servlet-mapping>
+	
+	<servlet>
+		<description>Fsim Admin Servlet</description>
+		<display-name>Fsim Admin Servlet</display-name>
+		<servlet-name>adminServlet</servlet-name>
+		<servlet-class>com.idca.fsim.web.AdminServlet</servlet-class>
+		
+		<init-param>	
+			<description>
+				When an hosts session is not an admin session then
+				hosty is blocked, time is increased with this value in minutes.
+			</description>
+			<param-name>hostBlockedNoAdminPeneltyTime</param-name>
+			<param-value>1</param-value>
+		</init-param>
+		
+	</servlet>
+	<servlet-mapping>
+		<servlet-name>adminServlet</servlet-name>
+		<url-pattern>/admin</url-pattern>
+	</servlet-mapping>
+
+	<servlet>
+		<description>Fsim Redirect Servlet</description>
+		<display-name>Fsim Redirect Servlet</display-name>
+		<servlet-name>redirectServlet</servlet-name>
+		<servlet-class>com.idca.fsim.web.RedirectServlet</servlet-class>
+
+		<init-param>	
+			<description>when click is done send to this url.</description>
+			<param-name>pluginPageRedirectURL</param-name>
+			<param-value>/ppage</param-value>
+		</init-param>
+		<init-param>
+			<description>blocked time im mins when parameter(s) is wrong</description>
+			<param-name>hostBlockedParameterPeneltyTime</param-name>
+			<param-value>0</param-value>
+		</init-param>
+	</servlet>
+	<servlet-mapping>
+		<servlet-name>redirectServlet</servlet-name>
+		<url-pattern>/redirect</url-pattern>
+	</servlet-mapping>
+	
+	<servlet>
+		<description>Fsim Plugin Page Servlet</description>
+		<display-name>Fsim Plugin Page Servlet</display-name>
+		<servlet-name>pluginPageServlet</servlet-name>
+		<servlet-class>com.idca.fsim.web.PluginPageServlet</servlet-class>
+		
+		<init-param>	
+			<description>blocked time im mins when parameter(s) is wrong</description>
+			<param-name>hostBlockedParameterPeneltyTime</param-name>
+			<param-value>0</param-value>
+		</init-param>
+	</servlet>
+	<servlet-mapping>
+		<servlet-name>pluginPageServlet</servlet-name>
+		<url-pattern>/ppage</url-pattern>
+	</servlet-mapping>
+
+
+	<!-- 
+	===================== FILTERS =================================
+	-->
+    <!-- 
+    	note: volgorde belangrijk !!
+    	aangezien servlet andere resources include bv !!
+    	en session of de aanwezig is !!
+    	
+    	Na de SessionFilter is er altijd een session beschikbaar.
+    	
+    	NU: request -> RequestLoggerFilter -> HostMonitoringFilter -> PagePathFilter -> SessionFilter -> AutenticationFilter -> RequestPerTimeFilter -> page(s)
+    -->
+
+	<filter>
+        <description>Manages user inlog/output</description>
+        <filter-name>authenticationFilter</filter-name>
+        <filter-class>com.idca.fsim.web.filters.AuthenticationFilter</filter-class>
+		<init-param>
+			<description>The page where a user comes when he logs out.</description>
+			<param-name>logoutRedirectURL</param-name>
+			<param-value>/search</param-value>
+		</init-param>
+    </filter>
+    <filter-mapping>
+        <filter-name>authenticationFilter</filter-name>
+        <url-pattern>/*</url-pattern>
+    </filter-mapping>
+
+
+</web-app>

WebRoot/css/fsim-layout-default.css

@@ -0,0 +1,54 @@
+/* ---- THIS STYLE SHEETS ONLY HANDLES STYLE CLASSES --- */
+
+
+
+#top {
+	height:		100px;	
+}
+
+#topLogo {
+	float:		left;
+	width:		150px;
+	height:		95%;
+}
+#topSearchBox {
+	float:		left;
+	width:		450px;
+	/*height:		95%;*/
+}
+
+#topFsimLinks {
+	float:		right;
+	width:		15%;
+	height:		95%;
+}
+
+#topBar {
+	
+}
+
+#results {
+	min-height:		100px;
+}
+
+#bottomBar {
+}
+
+#bottom {
+}
+
+#fsimKeywordsPopup {
+	position:	absolute;
+	width:		365px;
+	left:		160px;
+	top:		68px;
+	visibility:	hidden;
+	z-index:	1;
+
+	/* strange it wont do the class ??? */
+	font-size: 		60%;
+	border-style:	solid;
+	border-color:	green;
+	border-width:	2px;
+	background-color:#C4D8FD;
+}

WebRoot/css/fsim-style-default.css

@@ -0,0 +1,140 @@
+/* ---- THIS STYLE SHEETS ONLY HANDLES STYLE CLASSES --- */
+
+
+* {
+	margin:			0;
+	padding:		0;
+	font-family:	arial,sans-serif;
+}
+
+body {
+	/* nice border around site */
+	margin:			5px;	
+}
+
+img {
+	border:			0px;	
+}
+
+/* makes div visable for devv */
+/*
+div,span {
+	border:	1px solid #f00;
+	margin:	1px;
+}
+*/
+
+.top {
+	
+}
+
+.topSearchBox {
+	padding-left:	5px;
+	padding-right:	5px;
+}
+
+.topOptions {
+	font-size:	12pt;
+	color:		#aa6ea6;
+}
+
+.topBar {
+	background-color:	#C4D8FD;
+
+	border-top-style:	solid;
+	border-top-width:	1px;
+	border-top-color:	#FE3061;
+
+	border-bottom-style:solid;
+	border-bottom-width:1px;
+	border-bottom-color:#B4C8DD;
+}
+
+.topBarSearchStats {
+	font-size:		90%;
+	text-align:		right;
+}
+
+.topSearchBoxTitle {
+	text-align:		center;
+	font-weight:	bold;
+}
+
+.results {
+	width:			100%;
+	max-width:		100%;
+}
+
+.resultsURL {
+	font-size:		80%;
+}
+
+.resultsKeywordHighLight {
+	font-size:		120%;
+	color:				#FE3061;
+	font-weight:	bold;
+}
+
+.resultDetails {
+	font-size:		70%;
+	color:			#018001;
+}
+
+.bottomBar {
+	background-color:	#C4D8FD;
+
+	border-top-style:	solid;
+	border-top-width:	1px;
+	border-top-color:	#FE3061;
+
+	border-bottom-style:solid;
+	border-bottom-width:1px;
+	border-bottom-color:#FE3061;
+
+	padding:			2px;
+}
+
+.bottomBarPages {
+	text-align:		center;
+	font-weight:	bold;
+}
+
+.bottomBarRender {
+	font-size:		90%;
+	text-align:		right;
+}
+
+.bottom {
+}
+
+.bottomCopy {
+	font-size:	66%;
+	text-align:	center;
+}
+
+
+/* see the ID in layout this didn't work :? */
+.fsimKeywordsPopup {
+}
+
+.fsimCompleteBinderNormal {
+	/* height:				14px; */
+	background-color:	#C4D8FD;
+}
+.fsimCompleteBinderHighlight {
+	/* height:				14px; */
+	background-color:	#FE3061;
+}
+
+.fsimCompleteWords {
+	width:		80%;
+	font-size: 	90%;
+	/* float:		left; */
+}
+
+.fsimCompleteHits {
+	/* float:		right; */
+	width:		15%;
+	font-size: 	10px;
+	color:		#018001;
+}

WebRoot/denied/host_blocked.jsp

@@ -0,0 +1,54 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>HOST BLOCKED</title>
+	</head>
+	<body>
+		<h1>HOST BLOCKED</h1>
+		<hr/>
+		<hr/>
+		<br/>
+		You IP: <c:out value="${param['ip']}"/> has been
+		<c:if test="${param['perm'] == null}">
+			<form>
+			<span style="color:red">temperaly blocked.</span><br/>
+			Time to unblock:&nbsp;&nbsp;<input type="textfield" name="time" disabled/>&nbsp;&nbsp;in ms.<br/>
+			<br/>
+			<br/>
+			</form>
+		</c:if>
+		<c:if test="${param['perm'] != null}">
+			You are <span style="color:red">permenent blocked.</span><br/>
+			<br/>
+		</c:if>
+			Please consult the IDCA crew howto unblock if u think its not your fault.<br/>
+			(Bring some cold drinks. ;-)<br/>
+		<br/>
+		<hr/>
+		<hr/>
+		<br/>
+		Last reason: <c:out value="${param['reason']}"/>
+		<br/>
+		
+		<script type="text/javascript">	
+			var time = <c:out value="${param['time']}"/>+30000;
+			var obj = document.forms[0].elements[0];
+			var timeInterval = 1795;
+			doTime();
+			
+			function doTime()
+			{
+				time = time - timeInterval;
+				if(time>0)
+				{
+					obj.value = time;
+					setTimeout("doTime()",timeInterval);
+					return;
+				}
+				window.location='/search';
+			}
+		</script>
+	</body>
+</html>

WebRoot/denied/max_sessions.jsp

@@ -0,0 +1,19 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Max users reached.</title>
+	</head>
+	<body>
+		<br/>
+		<hr/>
+		<br/>
+		Sorry, but our max concurrent users limit is rearched.<br/>
+		<br/>
+		Please try again in a 5 minutes.<br/>
+		<br/>
+		<hr/>
+		<br/>
+	</body>
+</html>

WebRoot/denied/real_error.jsp

@@ -0,0 +1,20 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Real error encountered.</title>
+	</head>
+	<body>
+		<br/>
+		<hr/>
+		<br/>
+		Sorry, but there was een error.<br/>
+		<br/>
+		Please try again.<br/>
+		<a href="/search">FSIM</a>
+		<br/>
+		<hr/>
+		<br/>
+	</body>
+</html>

WebRoot/fragments/fsim_bottom.jsp

@@ -0,0 +1,107 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+		<!-- START BOTTOM -->
+		</div>	
+		<div id="bottomBar" class="bottomBar">&nbsp;
+			<c:if test="${drawPageResults == true}">
+				<div class="bottomBarPages">
+					<c:forEach var="page" items="${searchBean.pages}">
+						<a href="<c:url value="/search">
+									<c:param name="m" value="0"/>
+									<c:param name="p" value="${page}"/>
+									<c:param name="q" value="${searchBean.searchQuery}"/>
+								</c:url>"><c:out value="${page}"/></a>
+						&nbsp;
+					</c:forEach>
+				</div>
+				<div class="bottomBarRender">
+					<fmt:message key="fsim.bottombar.rendered" bundle="${fsim}"/>
+					<c:out value="${renderTime}"/>
+					<fmt:message key="fsim.bottombar.inms" bundle="${fsim}"/>
+				</div>
+			</c:if>
+		</div>
+		
+		<div id="bottom" class="bottom">
+			<div class="bottomCopy">
+				<br/>
+				FSIM: Is a pluggable DataBae search engine.<br/>
+				Very customizable. To fit all needs.<br/>		
+				<br/>
+				<fmt:message key="fsim.copyright" bundle="${fsim}"/>
+			</div>
+		</div>
+		<div id="debug">
+		</div>
+		<div id="tech" class="tech">
+			<a	href="http://jigsaw.w3.org/css-validator/"
+				title="<fmt:message key="fsim.tech.vcss.title" bundle="${fsim}"/>"
+				>
+				<img	src="http://jigsaw.w3.org/css-validator/images/vcss"
+						height="31"
+						width="88"
+						alt="<fmt:message key="fsim.tech.vcss.alt" bundle="${fsim}"/>"
+					/>
+			</a>
+			
+			<a	href="http://validator.w3.org/check?uri=referer"
+				title="<fmt:message key="fsim.tech.xhtml11.title" bundle="${fsim}"/>"
+				>
+				<img	src="/images/tech/valid-xhtml11.png"
+						height="31"
+						width="88"
+						alt="<fmt:message key="fsim.tech.xhtml11.alt" bundle="${fsim}"/>"
+					/>
+			</a>
+				
+          	<a	href="http://www.debian.org"
+          		title="<fmt:message key="fsim.tech.debian.title" bundle="${fsim}"/>"
+          		>
+          		<img	src="/images/tech/debian.gif"
+          				alt="<fmt:message key="fsim.tech.debian.alt" bundle="${fsim}"/>"
+          		/>
+          	</a>
+          	
+          	<a	href="http://sun.java.com"
+          		title="<fmt:message key="fsim.tech.java.title" bundle="${fsim}"/>"
+          		>
+          		<img	src="/images/tech/duke.gif"
+          				height="60"
+          				width="60"
+          				alt="<fmt:message key="fsim.tech.java.alt" bundle="${fsim}"/>"
+          			/>
+          	</a>
+          	
+          	<a	href="http://www.postgres.org"
+          		title="<fmt:message key="fsim.tech.postgres.title" bundle="${fsim}"/>"
+          		>
+          		<img	src="/images/tech/postgres.gif"
+          				height="50"
+          				width="50"
+          				alt="<fmt:message key="fsim.tech.postgres.alt" bundle="${fsim}"/>"
+          			/>
+          	</a>
+          	
+          	<a	href="http://www.kernel.org"
+          		title="<fmt:message key="fsim.tech.linux.title" bundle="${fsim}"/>"
+          		>
+          		<img	src="/images/tech/tux.gif"
+          				height="60"
+          				width="60"
+          				alt="<fmt:message key="fsim.tech.linux.alt" bundle="${fsim}"/>"
+          			/>
+          	</a>
+          	
+          	<a	href="http://jakarta.apache.org/tomcat/"
+          		title="<fmt:message key="fsim.tech.tomcat.title" bundle="${fsim}"/>"
+          		>
+          		<img	src="/images/tech/tomcat.gif"
+          				height="50"
+          				width="60"
+          				alt="<fmt:message key="fsim.tech.tomcat.alt" bundle="${fsim}"/>"
+          			/>
+          	</a>
+          	
+		</div>
+	</body>
+</html>	

WebRoot/fragments/fsim_httphosts.jsp

@@ -0,0 +1,51 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt" prefix="fmt" %>
+<div class="fsimHttpHosts">
+	<br/>
+	<form action="/admin" method="POST">
+		IP: <input type="textfield" name="host_address"/>
+		Reason: <input type="textfield" name="host_reason"/>
+		<input type="submit" name="host_add" value="add"/>
+		<input type="hidden" name="m" value="1"/>
+	</form>
+	<br/>
+	<table border="1">
+		<thead>
+			<tr>
+				<td>IP</td>
+				<td>startDate</td>
+				<td>lastMessage</td>
+				<td>blocked</td>
+				<td>blockedTime</td>
+				<td>lastRequest</td>
+				<td>permenent</td>
+			</tr>
+		</thead>
+		<tbody>
+			<c:forEach var="host" items="${httpHosts}">
+				<tr>
+					<td><c:out value="${host.hostAddress}"/>
+						<form action="/admin" method="POST">
+							<input type="submit" name="host_monitoring_remove" value="remove"/>
+							<input type="hidden" name="m" value="1"/>
+							<input type="hidden" name="host_address" value="<c:out value="${host.hostAddress}"/>"/>
+						</form>
+					</td>
+					<td><c:out value="${host.monitoredStartDate}"/></td>
+					<td><c:out value="${host.lastMessage}"/></td>
+					<td><c:out value="${host.blocked}"/></td>
+					<td><c:out value="${host.blockedTime}"/></td>
+					<td><c:out value="${host.lastRequest}"/></td>
+					<td><c:out value="${host.permenent}"/>
+						<form action="/admin" method="POST">
+							<input type="submit" name="host_permanent_add" value="set permanent"/>
+							<input type="submit" name="host_permanent_remove" value="remove permanent"/>
+							<input type="hidden" name="m" value="1"/>
+							<input type="hidden" name="host_address" value="<c:out value="${host.hostAddress}"/>"/>
+						</form>
+					</td>
+				</tr>
+			</c:forEach>
+		</tbody>
+	</table>
+</div>	

WebRoot/fragments/fsim_sessions.jsp

@@ -0,0 +1,43 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt" prefix="fmt" %>
+<%@ page import="com.idca.fsim.web.session.SearchBean" %>
+<%@ page import="java.util.Enumeration" %>
+<div class="fsimSessions">
+	<table border="1">
+		<thead>
+			<tr>
+				<td width="15%">SESSIONID</td>
+				<td width="10%">created</td>
+				<td width="10%">last access</td>
+				<td>objects</td>
+			</tr>
+		</thead>
+		<tbody>
+			<c:forEach var="session2" items="${sessions}">
+				<tr>
+					<td><c:out value="${session2.id}"/></td>
+					<td><c:out value="${session2.creationTime}"/></td>
+					<td><c:out value="${session2.lastAccessedTime}"/></td>
+					<td>
+						<%
+							Enumeration sessionNames = session.getAttributeNames();
+							while (sessionNames.hasMoreElements()) {
+								String sessionName = (String)sessionNames.nextElement();
+								String className = session.getAttribute( sessionName ).getClass().getName();
+								out.println(sessionName+"("+className+")<br/>");
+								
+								Object object = session.getAttribute(sessionName);
+								if(object instanceof SearchBean)
+								{
+									SearchBean sb = (SearchBean)object;
+									out.println("searched: "+sb.getSearchQuery()+"<br/>");
+								}
+							}
+						
+						%>
+					</td>
+				</tr>
+			</c:forEach>
+		</tbody>
+	</table>
+</div>

WebRoot/fragments/fsim_sources.jsp

@@ -0,0 +1,48 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt" prefix="fmt" %>
+<div class="fsimSessions">
+	<table border="1">
+		<thead>
+			<tr>
+				<td>name</td>
+				<td>table</td>
+				<td>search column</td>
+				<td>last index</td>
+				<td>hits</td>
+				<td>last id</id>
+			</tr>
+		</thead>
+		<tbody>
+			<c:forEach var="source" items="${sources}">
+				<tr>
+					<td>
+						<c:out value="${source.name}"/>
+						<form action="/admin" method="POST">
+							<input type="submit" name="source_index" value="index"/>
+							<input type="hidden" name="m" value="2"/>
+							<input type="hidden" name="source_name" value="<c:out value="${source.name}"/>"/>
+						</form>
+					</td>
+					<td><c:out value="${source.tableName}"/></td>
+					<td><c:out value="${source.searchColumnName}"/></td>
+					<td><c:out value="${source.lastIndexed}"/></td>
+					<td><c:out value="${source.hitsSize}"/></td>
+					<td><c:out value="${source.lastIndexedHitID}"/></td>
+				</tr>
+			</c:forEach>
+		</tbody>
+	</table>
+	<br/>
+	<br/>
+	Update path cache:
+	<form action="/admin" method="POST">
+		<input type="submit" name="path_update" value="path_update"/>
+		<input type="hidden" name="m" value="2"/>
+	</form>
+	<br/>
+	Dump wordLists info in log.
+	<form action="/admin" method="POST">
+		<input type="submit" name="wordlists_info" value="wordlists info"/>
+		<input type="hidden" name="m" value="2"/>
+	</form>
+</div>

WebRoot/fragments/fsim_top.jsp

@@ -0,0 +1,137 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt" prefix="fmt" %>
+<%@ page import="java.util.List" %>
+<%@ page import="java.util.Collection" %>
+<%@ page import="java.util.Iterator" %>
+<%@ page import="java.net.URLEncoder" %>
+<%@ page import="com.idca.fsim.core.plugins.SearchSource" %>
+<fmt:setBundle basename="com.idca.fsim.resources.i18n.fsim" var="fsim" scope="request"/>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title><fmt:message key="fsim.title" bundle="${fsim}"/></title>
+		<script type="text/javascript"
+				src="/js/fsim.js">
+		</script>
+		<link 	rel="stylesheet"
+				type="text/css"
+				href="/css/fsim-layout-default.css"
+				media="all"
+			/>
+		<link 	rel="stylesheet"
+				type="text/css"
+				href="/css/fsim-style-default.css"
+				media="screen"
+			/>
+		<meta 	http-equiv="Content-Type"
+				content="text/html; charset=UTF-8"
+			/>
+		<meta 	name="pagename"
+				content="FSIM: File Search Index Machine"
+			/>
+		<meta 	name="description"
+				content="File Search Indexing Machine"
+			/>
+		<meta 	name="keywords"
+				content="fsim,idca,search,searching,file,files,directory,path,browse,ftp"
+			/>
+		<meta 	name="robots"
+				content="all"
+			/>
+		<meta 	name="author"
+				content="http://www.idca.nl"
+			/>
+	</head>
+	<body onload="javascript:createAutoComplete();">
+		<div id="top">
+		<form action="/search" method="POST">
+			<div id="topLogo" class="toplogo">
+				<a	href="/search"
+					title="<fmt:message key="fsim.logo.title" bundle="${fsim}"/>"
+					>
+					<img	src="images/fsim.png"
+							alt="<fmt:message key="fsim.logo.alt" bundle="${fsim}"/>"
+						/>
+				</a>
+			</div>
+			<c:if test="${drawSearch == true}">
+				<div id="topSearchBox" class="topSearchBox">
+					<div class="topSearchBoxTitle"><br/><fmt:message key="fsim.header.title" bundle="${fsim}"/></div>
+					<div>
+						<input	name="q"
+								id="q"
+								size="35"
+								autocomplete="off"
+								type="text"
+								value="<c:out value="${searchBean.searchQuery}"/>"/>
+						&nbsp;
+						<input	type="submit"
+								value="<fmt:message key="fsim.search.button" bundle="${fsim}"/>"/>
+						<div id="fsimKeywordsPopup"></div>
+					</div>
+				</div>
+				<div id="topFsimLinks" class="topFsimLinks">
+					<br/>
+					Beta version.<br/>
+					<%
+						//<c:forEach var="source" items="${searchSources}">
+						Collection ss = (Collection)request.getAttribute("searchSources2");
+						Iterator i = ss.iterator();
+						while(i.hasNext())
+						{
+							request.setAttribute("source",i.next());
+							
+					%>
+						<input	type="checkbox"
+								name="<c:out value="${source.name}"/>"
+								onchange="this.form.submit();"
+								<%
+									String sss = (String)session.getAttribute("searchSources");
+									SearchSource ss1 = (SearchSource)request.getAttribute("source");
+									System.out.println("ssss: "+sss+" sdfsdf: " + ss1.getName());
+									if(sss.contains(ss1.getName())) {
+										out.print(" checked=\"true\" ");
+									}
+							
+							%>
+							/>
+								<c:out value="${source.name}"/>
+						<br/>
+					<%
+					
+					}
+						//</c:forEach>
+					%>
+				</div>
+			</c:if>
+		</div>
+	
+		<c:if test="${admin == true}">
+			Welcome admin user&nbsp;
+			<a href="/admin?m=0">sessions</a>&nbsp;<a href="/admin?m=1">host monitoring</a>&nbsp;<a href="/admin?m=2">sources</a>
+			<br/>
+		</c:if>	
+	
+		<div id="topBar" class="topBar">&nbsp;
+			<c:if test="${drawPageResults == true}">
+				<div class="bottomBarPages">
+					<c:forEach var="page" items="${searchBean.pages}">
+						<a href="<c:url value="/search">
+									<c:param name="m" value="0"/>
+									<c:param name="p" value="${page}"/>
+									<c:param name="q" value="${searchBean.searchQuery}"/>
+								</c:url>"><c:out value="${page}"/></a>
+						&nbsp;
+					</c:forEach>
+				</div>
+				<div class="topBarSearchStats"><fmt:message key="fsim.topbar.results" bundle="${fsim}"/>
+					<c:out value='${resultSize}'/> <fmt:message key="fsim.topbar.resultsin" bundle="${fsim}"/>
+					<c:out value='${searchTime}'/> <fmt:message key="fsim.topbar.inms" bundle="${fsim}"/>				
+				</div>	
+			</c:if>
+		</div>
+	</form>
+	</div>
+	<div id="results">
+	<br/>
+	<!-- END TOP -->

WebRoot/fragments/searchers/files/file_mirrors.jsp

@@ -0,0 +1,4 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+
+SEE MIRROR LIST

WebRoot/fragments/searchers/files/file_redirect.jsp

@@ -0,0 +1,24 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+<%@ page import="com.idca.fsim.searchers.files.models.File" %>
+<%@ page import="com.idca.fsim.searchers.files.models.Path" %>
+<%@ page import="java.net.URLEncoder" %>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<%
+	File file = (File)session.getAttribute("model");
+	String url = file.getPath().getPath()+"/"+file.getName();	
+	url = file.getPath().getHost().getProtocol()+file.getPath().getHost().getHostName()+url;//URLEncoder.encode(url, "UTF-8");
+	request.setAttribute("url",url);
+%>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<META http-equiv=Refresh Content="0;URL=<c:out value="${url}"/>"/>
+	</head>
+	<body>
+		Going to path now.
+	</body>
+</html>
+
+
+

WebRoot/fragments/searchers/files/host_browse.jsp

@@ -0,0 +1,39 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+<%@ page import="java.util.Collection" %>
+<%@ page import="com.idca.fsim.searchers.files.models.Host" %>
+<%@ page import="com.idca.fsim.searchers.files.FilesListener" %>
+<%@ page import="com.idca.fsim.searchers.files.PathManager" %>
+<%
+	Collection hosts = FilesListener.getPathManager().getHosts();
+	request.setAttribute("hosts",hosts);
+%>
+<table>
+	<thead>
+		<tr>
+			<td></td>
+			<td></td>
+			<td></td>
+		</tr>
+	</thead>
+	<tbody>
+		<c:forEach var="host" items="${hosts}">
+			<tr>
+				<td>&gt; <c:out value="${host.hostname}"/></td>
+				<td>
+					<span class="resultDetails">
+					<%
+						Host host = (Host)request.getAttribute("host");
+						out.println(host.getRootPaths().size());
+					%>
+					</span>
+				</td>
+				<td>
+					<span class="resultDetails">
+						<c:out value="${host.lastScanned}"/>
+					</span>
+				</td>
+			</tr>
+		</c:forEach>
+	</tbody>
+</table>

WebRoot/fragments/searchers/files/host_redirect.jsp

@@ -0,0 +1,19 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+<%@ page import="com.idca.fsim.searchers.files.models.Host" %>
+<%@ page import="java.net.URLEncoder" %>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<%
+	Host host = (Host)session.getAttribute("model");
+	String url = host.getProtocol()+host.getHostName();
+	//url = URLEncoder.encode(url, "UTF-8");
+	request.setAttribute("url",url);
+%>
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<META http-equiv=Refresh Content="0;URL=<c:out value="${url}"/>"/>
+	</head>
+	<body>
+		Going to host now.
+	</body>
+</html>

WebRoot/fragments/searchers/files/path_browse.jsp

@@ -0,0 +1,150 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+<%@ page import="java.util.List" %>
+<%@ page import="java.util.Collection" %>
+<%@ page import="java.util.Iterator" %>
+<%@ page import="java.util.ArrayList" %>
+<%@ page import="java.net.URLEncoder" %>
+<%@ page import="com.idca.fsim.searchers.files.models.Host" %>
+<%@ page import="com.idca.fsim.searchers.files.models.File" %>
+<%@ page import="com.idca.fsim.searchers.files.models.Path" %>
+<%@ page import="com.idca.fsim.searchers.files.FilesListener" %>
+<%
+	Object model = session.getAttribute("model");
+	Path path = null;
+	Host host = null;
+	
+	Collection paths = null;
+	List files = null;
+	
+	if(model instanceof File)
+	{
+		File file = (File)model;
+		path = file.getPath();
+		paths = path.getChilderen();
+		files = FilesListener.getPathManager().getFiles(path);
+	}
+	
+	if(model instanceof Path)
+	{
+		path = (Path)model;
+		paths = path.getChilderen();
+		files = FilesListener.getPathManager().getFiles(path);
+	}
+	if(model instanceof Host)
+	{
+		host = (Host)model;
+		if (host.getRootPaths().iterator().hasNext()) {
+			paths = ((Path)(host.getRootPaths().iterator().next())).getChilderen();
+			files = FilesListener.getPathManager().getFiles(((Path)(host.getRootPaths().iterator().next())));
+		} else {
+			paths = new ArrayList();
+			files = new ArrayList();
+		}
+	}
+	
+	request.setAttribute("paths",paths);
+	request.setAttribute("files",files);
+%>
+<table width="100%">
+	<thead>
+		<tr>
+			<td width="45%"></td>
+			<td width="25%"></td>
+			<td width="25%"></td>
+		</tr>
+	</thead>
+	<tbody>
+		<%  if (path!=null && path.getParent()!=null) {  %>
+				<tr>
+				<td><span class="resultDetails"><a href="<%
+						out.write("ppage?ss="+request.getParameter("ss")+"&rid="+path.getParent().getID()+"&up=path_browse");			
+					%>">..</a></span></td>
+				<td><span class="resultDetails"></span></td>
+				<td><span class="resultDetails"></span></td>
+			</tr>
+		<%  }  %>
+	
+	
+	
+	
+			 <% if (host!=null) {
+				%>
+				<tr>
+				<td><span class="resultDetails">/</span></td>
+				<td><span class="resultDetails">(root)</span></td>
+				<td><span class="resultDetails"></span></td>
+			</tr>
+		<% }
+	
+		//<c:forEach var="path" items="${paths}">
+		Iterator p = paths.iterator();
+		while(p.hasNext())
+		{
+			Path pp = (Path)p.next();
+			request.setAttribute("path",pp);
+	%>
+			<tr>
+				<td>&gt; <a href="<c:out value="${path.path}"/>"><c:out value="${path.path}"/></a></td>
+				<td>
+					<span class="resultDetails">
+					<%
+						Path path2 = (Path)request.getAttribute("path");
+						//if(path2!=null) {
+							out.println(path2.getChilderen().size());
+						//} else {
+						// out.println("path2 null");
+						 //}
+						
+						
+					%>
+					</span>
+				</td>
+				<td>
+					<span class="resultDetails">
+						<c:out value="${path.host.lastScanned}"/>
+					</span>
+				</td>
+			</tr>
+<%			}
+		
+		//</c:forEach
+		//<c:forEach var="file" items="${files}">
+		Iterator f = files.iterator();
+		while(f.hasNext())
+		{
+			request.setAttribute("file",f.next());
+			File file = (File)request.getAttribute("file");
+			
+			
+			//redirect?ss="+hit.source.getName()+"&rid="+hit.id+"&up=path_browse"+"\">browse");
+			
+			String url = "redirect/"+file.getPath()+file.getName();
+			
+			url = file.getPath().getHost().getProtocol()+file.getPath().getHost().getHostName()+URLEncoder.encode(url, "UTF-8");
+			
+		%>
+			<tr>
+				<td><span class="resultDetails"><a href="<%=url%>"><c:out value="${file.name}"/></a></span></td>
+				<td><span class="resultDetails">
+					<%
+						//File file = (File)request.getAttribute("file");
+						//if(file!=null) {
+							out.println(File.printSize(file));
+						//} else {
+						//	out.println("file is null");
+						//}
+					%>
+					</span>
+				</td>
+				<td>
+					<span class="resultDetails">
+					</span>
+				</td>
+			</tr>
+		<%
+		}
+			//</c:forEach>
+		%>
+	</tbody>
+</table>

WebRoot/fragments/searchers/files/path_redirect.jsp

@@ -0,0 +1,22 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"   %>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt"  prefix="fmt" %>
+<%@ page import="com.idca.fsim.searchers.files.models.File" %>
+<%@ page import="com.idca.fsim.searchers.files.models.Path" %>
+<%@ page import="java.net.URLEncoder" %>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<%
+	Path path = (Path)session.getAttribute("model");
+	String url = path.getPath();
+	//url = URLEncoder.encode(url, "UTF-8");
+	request.setAttribute("url",path.getHost().getProtocol()+path.getHost().getHostName()+url);
+	//<jsp:useBean id="hit" class="com.idca.fsim.models.Hit" scope="session"/>
+	//<jsp:useBean id="model" class="com.idca.fsim.searchers.files.models.Path" scope="session"/>
+%>
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<META http-equiv=Refresh Content="0;URL=<c:out value="${url}"/>"/>
+	</head>
+	<body>
+		Downloading file now.
+	</body>
+</html>

WebRoot/images/tech/.svn/all-wcprops

@@ -0,0 +1,47 @@
+K 25
+svn:wc:ra_dav:version-url
+V 56
+/svn/idca/!svn/ver/3/fsim/trunk/fsim/WebRoot/images/tech
+END
+duke.gif
+K 25
+svn:wc:ra_dav:version-url
+V 65
+/svn/idca/!svn/ver/3/fsim/trunk/fsim/WebRoot/images/tech/duke.gif
+END
+vcss.png
+K 25
+svn:wc:ra_dav:version-url
+V 65
+/svn/idca/!svn/ver/3/fsim/trunk/fsim/WebRoot/images/tech/vcss.png
+END
+tomcat.gif
+K 25
+svn:wc:ra_dav:version-url
+V 67
+/svn/idca/!svn/ver/3/fsim/trunk/fsim/WebRoot/images/tech/tomcat.gif
+END
+debian.gif
+K 25
+svn:wc:ra_dav:version-url
+V 67
+/svn/idca/!svn/ver/3/fsim/trunk/fsim/WebRoot/images/tech/debian.gif
+END
+tux.gif
+K 25
+svn:wc:ra_dav:version-url
+V 64
+/svn/idca/!svn/ver/3/fsim/trunk/fsim/WebRoot/images/tech/tux.gif
+END
+valid-xhtml11.png
+K 25
+svn:wc:ra_dav:version-url
+V 74
+/svn/idca/!svn/ver/3/fsim/trunk/fsim/WebRoot/images/tech/valid-xhtml11.png
+END
+postgres.gif
+K 25
+svn:wc:ra_dav:version-url
+V 69
+/svn/idca/!svn/ver/3/fsim/trunk/fsim/WebRoot/images/tech/postgres.gif
+END

WebRoot/images/tech/.svn/entries

@@ -0,0 +1,266 @@
+9
+
+dir
+115
+https://decaid.idca.nl/svn/idca/fsim/trunk/fsim/WebRoot/images/tech
+https://decaid.idca.nl/svn/idca
+
+
+
+2005-08-08T22:22:39.000000Z
+3
+willemc
+
+
+svn:special svn:externals svn:needs-lock
+
+
+
+
+
+
+
+
+
+
+
+623d4b01-600f-0410-8ed7-89934cd2f745
+
+debian.gif
+file
+
+
+
+
+2009-08-11T18:38:40.000000Z
+8ff7f09386f6937371ff2e31e3bf5a7e
+2005-08-08T22:22:39.000000Z
+3
+willemc
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+2596
+
+duke.gif
+file
+
+
+
+
+2009-08-11T18:38:40.000000Z
+70263cae8e43a375c7f260b6d731b7ce
+2005-08-08T22:22:39.000000Z
+3
+willemc
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+6310
+
+postgres.gif
+file
+
+
+
+
+2009-08-11T18:38:40.000000Z
+3bdb0172fbc3e28067700488f4a8a1a0
+2005-08-08T22:22:39.000000Z
+3
+willemc
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+9296
+
+tomcat.gif
+file
+
+
+
+
+2009-08-11T18:38:40.000000Z
+b8ebd781db53b856efa8e873fa4d2f6e
+2005-08-08T22:22:39.000000Z
+3
+willemc
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+1934
+
+tux.gif
+file
+
+
+
+
+2009-08-11T18:38:39.000000Z
+e5b3348e57a2212dfac480d21074fa4c
+2005-08-08T22:22:39.000000Z
+3
+willemc
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+4972
+
+valid-xhtml11.png
+file
+
+
+
+
+2009-08-11T18:38:40.000000Z
+6b1146d15bc3a499567953020fd14204
+2005-08-08T22:22:39.000000Z
+3
+willemc
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+1917
+
+vcss.png
+file
+
+
+
+
+2009-08-11T18:38:39.000000Z
+35986567c8c46ef7fe2fd6f769f8eab3
+2005-08-08T22:22:39.000000Z
+3
+willemc
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+1134
+

WebRoot/images/tech/.svn/format

@@ -0,0 +1 @@
+9

WebRoot/images/tech/.svn/prop-base/debian.gif.svn-base

@@ -0,0 +1,5 @@
+K 13
+svn:mime-type
+V 24
+application/octet-stream
+END

WebRoot/images/tech/.svn/prop-base/duke.gif.svn-base

@@ -0,0 +1,5 @@
+K 13
+svn:mime-type
+V 24
+application/octet-stream
+END

WebRoot/images/tech/.svn/prop-base/postgres.gif.svn-base

@@ -0,0 +1,5 @@
+K 13
+svn:mime-type
+V 24
+application/octet-stream
+END

WebRoot/images/tech/.svn/prop-base/tomcat.gif.svn-base

@@ -0,0 +1,5 @@
+K 13
+svn:mime-type
+V 24
+application/octet-stream
+END

WebRoot/images/tech/.svn/prop-base/tux.gif.svn-base

@@ -0,0 +1,5 @@
+K 13
+svn:mime-type
+V 24
+application/octet-stream
+END

WebRoot/images/tech/.svn/prop-base/valid-xhtml11.png.svn-base

@@ -0,0 +1,5 @@
+K 13
+svn:mime-type
+V 24
+application/octet-stream
+END

WebRoot/images/tech/.svn/prop-base/vcss.png.svn-base

@@ -0,0 +1,5 @@
+K 13
+svn:mime-type
+V 24
+application/octet-stream
+END

WebRoot/index.jsp

@@ -0,0 +1,2 @@
+<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %>
+<c:redirect url="/search"/>

WebRoot/js/fsim.js

@@ -0,0 +1,253 @@
+
+
+	//if(window.url.indexOf("divv")!=-1) {	
+	//	alert("turning on divv devv mode");
+	//div,span {
+	//border:	1px solid #f00;
+	//margin:	1px;
+	//}
+	//}
+
+
+	var IE = document.all?true:false;
+	var req = null;
+	if (!IE) {
+		document.captureEvents(Event.KEYBOARD);
+	}
+	
+	// full autocomplete off AND XHTML compatible
+	for(i=0;i<document.forms.length;i++)void(document.forms[i].setAttribute('autocomplete', 'off'));
+
+	function debug(text) {
+	
+		var divObject = document.getElementById('debug');
+		divObject.innerHTML = divObject.innerHTML+text+'<br/>';
+	
+	}
+
+	function initRequest() {
+	    if (window.XMLHttpRequest) {
+	        req = new XMLHttpRequest();
+	    } else if (window.ActiveXObject) {
+    	    IE = true;
+	        req = new ActiveXObject("Microsoft.XMLHTTP");
+	    }
+	}
+	
+	function sendRequest(url,oDiv) {
+	
+		if(req.readyState==1) {
+			// not while loading.
+			return;	
+		}
+	
+		req.table = oDiv;
+		req.onreadystatechange=callBack;
+		req.open("GET",url,true);
+		req.send(null);
+
+	}
+	
+	function callBack()
+	{
+		// if xmlhttp shows "loaded"
+		if(req && req.readyState==4)
+		{
+  			// if "OK"
+			if (req.status==200)
+			{
+				//alert("just got: "+xmlhttp.responseText);
+				//document.write(xmlhttp.responseText);
+				ac = req.table;
+				oDiv = ac.oDiv;
+				
+				// clear the popup div.
+				while ( oDiv.hasChildNodes() )
+				{
+					oDiv.removeChild(oDiv.firstChild);
+				}
+	
+				var items = req.responseXML.getElementsByTagName("item");
+				//alert("just got items:"+items);
+				var to = "";
+			    for (loop = 0; loop < items.length; loop++)
+			    {
+			        var item = items.item(loop).firstChild.nodeValue; 
+			        var hits = items.item(loop).getAttribute('hits');
+					
+					var oDivBinder = document.createElement('div');
+					oDivBinder.id = loop;
+					oDivBinder.className = 'fsimCompleteBinderNormal';
+					
+					var oDivItem = document.createElement('div');					
+					oDivItem.innerHTML = item;
+					oDivItem.className = 'fsimCompleteWords';
+					
+					oDivBinder.onmousedown = AutoComplete.prototype.onDivMouseDown;
+					oDivBinder.onmouseover = AutoComplete.prototype.onDivMouseOver;
+					oDivBinder.onmouseout = AutoComplete.prototype.onDivMouseOut;
+					
+					//var oSpanHits = document.createElement('div');
+					//oSpanHits.className = 'fsimCompleteHits';
+					//oSpanHits.innerHTML = 'hits: '+hits;
+					
+					oDivBinder.appendChild(oDivItem);
+					//oDivBinder.appendChild(oSpanHits);
+					
+					oDiv.AutoComplete = ac;
+					oDiv.appendChild(oDivBinder);		
+				}
+				//alert("got: "+to);
+				oDiv.style.visibility = "visible";
+				//oDiv.stlyeClass = "pop";
+				oDiv.className = "pop";
+			}
+			else
+			{
+				alert("Problem retrieving XML data status:"+req.statusText);
+			}
+		}
+	}
+	
+	function AutoComplete(url, oText, oDiv)
+	{
+		// initialize member variables
+		this.oText = oText; // the text box
+		this.oDiv = oDiv; // a hidden <div> for the popup auto-complete
+		this.url=url;
+		this.selectedItem = null;
+		
+		this.preKeyword = "";
+
+		
+		initRequest();
+
+		//if(IE) {
+			// turn of autocompletion in IE for this field !
+			oText.autocomplete = 'off';
+		//}
+
+		// attach handlers to the text-box
+		oText.AutoComplete = this;
+		oText.onkeyup = AutoComplete.prototype.onTextChange;
+		oText.onblur = AutoComplete.prototype.onTextBlur;
+	}
+
+	AutoComplete.prototype.onTextBlur = function()
+	{
+		this.AutoComplete.onblur();
+	}
+
+	AutoComplete.prototype.onblur = function()
+	{
+		this.oDiv.style.visibility = "hidden";
+	}
+
+	AutoComplete.prototype.onTextChange = function(e)
+	{
+		if(IE) {
+			var e = window.event;
+		}
+		//debug('keycode='+e.keyCode);	
+		var keyCode = e.keyCode;
+		
+		if(32 === keyCode) {
+			this.AutoComplete.preKeyword = this.AutoComplete.oText.value;
+			return;
+		}
+		
+		if(40 === keyCode || 38 === keyCode) {
+
+			if(this.AutoComplete.selectedItem !== null) {
+				this.AutoComplete.selectedItem.className = "fsimCompleteBinderNormal";
+				var currentID = this.AutoComplete.selectedItem.id;
+				//debug("curID="+currentID);
+			}
+			var nextItem = null;
+			if(40 === keyCode) {
+				if(!currentID) {
+					var currentID = -1;
+				}
+				currentID++;
+				nextItem = this.AutoComplete.oDiv.childNodes.item(currentID);
+				nextItem.className = "fsimCompleteBinderHighlight";
+				this.AutoComplete.selectedItem = nextItem;
+				
+				this.AutoComplete.oText.value = this.AutoComplete.preKeyword+nextItem.firstChild.innerHTML;
+				
+			}
+			if(38 === keyCode) {
+				if(!currentID) {
+					var currentID = this.AutoComplete.oDiv.childNodes.lenght+1;
+				}
+				currentID--;
+				nextItem = this.AutoComplete.oDiv.childNodes.item(currentID);
+				nextItem.className = "fsimCompleteBinderHighlight";
+				this.AutoComplete.selectedItem = nextItem;
+				
+				this.AutoComplete.oText.value = this.parentNode.AutoComplete.preKeyword+nextItem.firstChild.innerHTML;
+			}
+			
+		} else {
+			this.AutoComplete.onchange();
+		}
+	}
+	
+	AutoComplete.prototype.selectRange = function (iStart, iLength)
+	{
+    	if (this.oText.createTextRange) {
+        	var oRange = this.oText.createTextRange();
+	        oRange.moveStart("character", iStart);
+	        oRange.moveEnd("character", iLength - this.textbox.value.length);
+    	    oRange.select();
+	    } else if (this.oText.setSelectionRange) {
+    	    this.oText.setSelectionRange(iStart, iLength);
+	    }
+
+    	this.oText.focus();
+	}
+	
+
+	AutoComplete.prototype.onchange = function()
+	{
+		var txt = this.oText.value;
+		if(""!=txt) {
+			this.selectedItem = null;
+			sendRequest(this.url+txt,this);
+		} else {
+			// hide !
+			this.oDiv.style.visibility = "hidden";
+			this.oText.focus();
+		}
+	}
+
+	AutoComplete.prototype.onDivMouseDown = function()
+	{
+		this.parentNode.AutoComplete.oText.value = this.parentNode.AutoComplete.preKeyword+this.firstChild.innerHTML;
+		this.parentNode.AutoComplete.selectedItem = null;
+	}
+
+	AutoComplete.prototype.onDivMouseOver = function()
+	{
+		if(this.parentNode.AutoComplete.selectedItem !== null) {
+				this.parentNode.AutoComplete.selectedItem.className = "fsimCompleteBinderNormal";
+		}
+		this.className = 'fsimCompleteBinderHighlight';
+		this.parentNode.AutoComplete.selectedItem = this;
+		
+		this.parentNode.AutoComplete.oText.value = this.parentNode.AutoComplete.preKeyword+this.firstChild.innerHTML;
+	}
+	
+	AutoComplete.prototype.onDivMouseOut = function()
+	{
+		this.className = 'fsimCompleteBinderNormal';
+	} 
+
+
+	function createAutoComplete()
+	{
+		new AutoComplete('/search?m=1&l=',
+			document.getElementById('q'),
+			document.getElementById('fsimKeywordsPopup'));
+	}
+		

doc/README.txt

@@ -0,0 +1,6 @@
+
+== FSIM ==
+
+The File Search Index Machine
+
+TEST COMMIT

fsimcrawl/.buildpath

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path=""/>
+	<classpathentry kind="con" path="org.trustudio.python.launching.PYTHON_CONTAINER"/>
+	<classpathentry kind="output" path=""/>
+</classpath>

fsimcrawl/fsimcrawl.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python2.4
+"""
+fsimcrawl.py - a crawler for fsim
+
+this file is really only the launcher for the crawler
+"""
+
+import sys
+
+from core import FsimCrawl
+
+def startup():
+    # here we start running fsimcrawl
+    #FsimCrawl.FsimCrawl().start()
+    pass
+
+if __name__=="__main__":
+    startup()

fsimcrawl/tests/__init__.py

@@ -0,0 +1,3 @@
+"""
+tests
+"""

fsimcrawl/tests/test_ftplib.py

@@ -0,0 +1,460 @@
+#!/usr/bin/env python2.4
+
+import sys
+reload (sys) # enables the setencoding
+sys.setdefaultencoding('utf-8')
+import os
+import socket
+socket.setdefaulttimeout(3600)
+from ftplib import FTP
+import Queue
+import threading
+import time
+import gzip
+import cPickle
+
+#from pyPgSQL import PgSQL
+import psycopg2
+
+class FsimDBPathException(Exception): pass
+
+class FsimDB(threading.Thread):
+    def __init__(self, host, dbq):
+	threading.Thread.__init__(self)
+	self.host = host
+	self.ip = socket.gethostbyname(host)
+	self.dbq = dbq
+	print time.strftime("%H:%M:%S"), "FsimDB: pgsql connect"
+	self.db = psycopg2.connect("dbname=fsim user=fsim host=localhost password=fsim")
+	self.db.set_isolation_level(0)
+        print time.strftime("%H:%M:%S"), "FsimDB: pgsql cursor"
+	self.curs = self.db.cursor()
+	print time.strftime("%H:%M:%S"), "FsimDB: update host and spider"
+	spiderid, hostid = self.updateHostAndSpider(self.host, self.ip)
+	self.spiderid = spiderid
+	self.hostid = hostid
+	#self.findRootId()
+	
+	self.curs.execute("COMMIT;")
+	
+    	self.__die = False
+        self.running = True
+
+	self.curs.execute("COMMIT;")
+
+
+    def die(self):
+	self.__die = True
+    
+    def shutdown(self):
+	self.running = False
+    
+    def cursor(self):
+        return self.db.cursor()
+        
+    def getHostId(self, host):
+        self.curs.execute("SELECT id FROM host WHERE host = '%s'" % (host))
+        retval = self.curs.fetchone()
+        if not retval:
+            return False
+        return int(retval[0])
+
+    def insertSpider(self):
+	self.curs.execute("INSERT INTO spider (udate) VALUES (now())")
+	self.curs.execute("SELECT currval('spider_id_seq')")
+	retval = self.curs.fetchone()
+	if not retval:
+		return False
+	return int(retval[0])
+	
+    def insertHost(self, hostname, ip, spiderid):
+	query = "INSERT INTO host (hostname, host, insert_date, last_spider_id) VALUES"
+	query += " ('%s', '%s', now(), %i)" % (hostname, ip, spiderid)
+	print time.strftime("%H:%M:%S"), "inserting host"
+	self.curs.execute(query)
+	print time.strftime("%H:%M:%S"), "inserting done"
+
+    def updateHost(self, hostid, spiderid):
+	query = "UPDATE host SET last_spider_id=%(spider_id)s WHERE id=%(host_id)s"
+	self.curs.execute(query, dict(spider_id=spiderid, host_id=hostid))
+    
+    def createRootDir(self,hostid):
+        query = "INSERT INTO path (host_id, directory) VALUES (%(host_id)s,%(dir_id)s)"
+	self.curs.execute(query, dict(host_id=hostid,dir_id="/"))
+
+    def updateHostAndSpider(self, hostname, ip):
+	spiderid = self.insertSpider()
+	print 'spiderid:', spiderid
+	hostid = self.getHostId(hostname)
+	if not hostid:
+	    print time.strftime("%H:%M:%S"), "insert host"
+            self.insertHost(hostname, ip, spiderid)
+	    print time.strftime("%H:%M:%S"), "select host"
+            hostid = self.getHostId(ip)
+	    print "got host:",hostid
+	    self.createRootDir(hostid)
+	else:
+	    print time.strftime("%H:%M:%S"), "update host"
+            self.updateHost(hostid, spiderid)
+	return spiderid, hostid
+
+    def findRoot(self,path):
+	path2 = "/".join(path)
+	if not path2: path2="/"
+
+	try:
+		path2 = path2.encode("utf-8","replace");
+	except:
+		return None;
+
+	query = "SELECT id FROM path WHERE host_id=%(host_id)s AND directory=%(dir)s"
+	self.curs.execute(query, dict(host_id=self.hostid,dir=path2))
+	rootid = self.curs.fetchone()
+	if not rootid:
+		raise FsimDBPathException, "Invalid path %s" % path2
+    	return rootid[0]
+        
+    def updateDir(self,rootid,path2,name):
+
+	path = "/".join(path2)
+	if name:
+		path = path+"/"+name
+
+	if not path:
+		path = "/"+name
+
+	try:
+		path = path.encode('utf-8','replace');
+	except:
+		print "Encode error in directory: ",path;
+		return;
+		#path = path.encode('utf-8','ignore')
+
+	query = "Select id from path where host_id=%(host_id)s and directory=%(dir)s"
+	self.curs.execute(query, dict(host_id=self.hostid,dir=path))
+	result = self.curs.fetchone()
+	if result: return
+
+	query = "INSERT INTO path (host_id, directory,parent_id) VALUES (%(host_id)s,%(dir)s,%(parent_id)s)"
+	#print time.strftime("%H:%M:%S"), "updateDir SQL:",query
+	self.curs.execute(query, dict(host_id=self.hostid,dir=path,parent_id=rootid))
+	
+    def updateFile(self, rootid, name, size):
+	#print time.strftime("%H:%M:%S"), "updateFile file:", rootid, name, size, "-"
+
+	try:
+		name = name.encode('utf-8','replace');
+	except:
+		print "Encode error in name: ",name ;
+		return;
+		#name.encode('utf-8','ignore');
+
+	query = "select id from file where path_id=%(path_id)s and name=%(name)s and size=%(size)s";
+	self.curs.execute(query, dict(path_id=rootid,name=name,size=long(size)));
+	result = self.curs.fetchone();
+	if result: return;
+
+	query = "INSERT INTO file (path_id, name, size, insert_date) VALUES (%(path_id)s, %(name)s, %(size)s, now())";
+	self.curs.execute(query, dict(path_id=int(rootid),name=name,size=long(size)));
+	query = "select id from file where name=%(name)s and size=%(size)s and path_id=%(path_id)s";
+	self.curs.execute(query, dict(name=name,size=long(size),path_id=int(rootid)));
+	new_file_id = self.curs.fetchone();
+
+	# Search for mirrors
+	query = "select id from file where name=%(name)s and size=%(size)s and path_id!=%(path_id)s";
+	self.curs.execute(query, dict(name=name,size=long(size),path_id=int(rootid)));
+	mirror_id = self.curs.fetchone();
+	if not mirror_id:
+		# no same file found.. do nothing
+		return;
+		
+	# get the new id
+	if not new_file_id:
+		print "ERROR: no file id found of new file";
+
+	# test if we need to add ourself
+	query = "select id from file_mirror where first_file_id=%(first_id)s";
+	self.curs.execute(query, dict(first_id=int(mirror_id[0])) );
+	result = self.curs.fetchone();
+	if not result:
+		print "Adding mirror record";
+		query = "insert into file_mirror (file_id,first_file_id) values (%(file_id)s,%(first_id)s)";
+		self.curs.execute(query, dict(file_id=int(mirror_id[0]),first_id=int(mirror_id[0])));
+	query = "insert into file_mirror (file_id,first_file_id) values (%(file_id)s,%(first_id)s)";
+	self.curs.execute(query, dict(file_id=int(new_file_id[0]),first_id=int(mirror_id[0])));
+	return;
+    
+    def process(self, hit):
+	type, path, name, size = hit;
+	#print time.strftime("%H:%M:%S"), "hit:",hit
+
+	rootid = self.findRoot(path);
+
+        if type=="DIR":
+		# hit is directory
+                self.updateDir(rootid,path,name);
+	else:
+                # hit is file
+		rootid = self.findRoot(path);
+		if not rootid: return;
+                self.updateFile(rootid, name, size);
+    
+    def runner(self):
+	c=0
+	while (not self.__die):
+	    hit = None
+	    try:
+		hit = self.dbq.get(timeout=0.2)
+            except Queue.Empty, e:
+		# queue is empty
+		#print time.strftime("%H:%M:%S"), "FsimDB: Queue empty waiting"
+		if (not self.running or self.__die):
+		    # queue is empty and we are done running the crawl
+		    break
+	    else:
+        	self.process(hit)
+		
+            if c<32:
+                c+=1
+            else:
+                print time.strftime("%H:%M:%S"), "FsimDB:", self.dbq.qsize(), hit
+                c=0
+                #self.curs.execute("COMMIT;")
+            
+    def run(self):
+	print time.strftime("%H:%M:%S"), "FsimDB: run"
+	try:
+	    self.runner()
+        except KeyboardInterrupt, err:
+	    print time.strftime("%H:%M:%S"), "KeyboardInterrupt:", str(err)
+	    self.dbthread.die()
+	self.curs.execute("COMMIT;")
+	self.db.close()
+
+class cpickle_test:
+    def __init__(self, ip):
+	self.ip = ip
+	self.dbq = Queue.Queue()
+	print "loading map"
+	f = ((os.path.isfile(ip) and file(ip)) or gzip.open(ip+".pickle"))
+	self.fmap = cPickle.load(f)
+	self.dbthread = FsimDB(self.ip, self.dbq)
+	
+    def crawl(self):
+	c=0
+	while (self.fmap):
+	    if (not self.dbthread.isAlive()):
+		break
+	    path, lines = self.fmap.popitem()
+	    print time.strftime("%H:%M:%S"), "FsimCrawler path: %s" % path
+	    pre, post = os.path.split(path)
+	    pre = pre.split("/")
+	    if not pre[0]=="":
+		pre.insert(0, "")
+	    # FIXME: dirty hack to ensure all directories exists
+	    for i in range(len(pre)):
+		self.dbq.put(["DIR", pre[:i], pre[i], 0])
+	    # insert directory
+	    self.dbq.put(["DIR", pre, post, 0])
+	    for line in lines:
+		#print len(pathlines), path, line
+        	fullmode, links, owner, group, size, month, day, hour, name = line
+		size = int(size)
+    		#isdir = fullmode[0]=='d'
+		isdir = 0
+    	    	islink = fullmode[0]=='l'
+	    	#mode = fullmode[1:]
+
+		if (c<256):
+		    c+=1
+        	else:
+        	    print time.strftime("%H:%M:%S"), "FsimCrawler:", len(self.fmap)
+            	    c=0
+		
+		if name=="." or name=="..":
+		    continue
+	    
+        	#if (isdir):
+		#    # record directory and scanit
+		#    print time.strftime("%H:%M:%S"), "FsimCrawler path: %s/%s" % ("/".join(path), name)
+		#    self.dbq.put(["DIR", path, name, size])
+        	if (islink):
+		    # ignoring links FIXME later
+		    pass
+        	else:
+            	    # is file
+		    #print time.strftime("%H:%M:%S"), "found file:", "/".join(path), name
+		    self.dbq.put(["FILE", pre+[post], name, size])
+	
+    def run(self):
+        print time.strftime("%H:%M:%S"), "FsimCrawl: run"
+        # start test
+        self.dbthread.start()
+
+	try:
+            self.crawl()
+        except KeyboardInterrupt, err:
+	    self.dbthread.die()
+	    print time.strftime("%H:%M:%S"), "KeyboardInterrupt:", str(err)
+        except Exception, e:
+	    self.dbthread.die()
+	    print time.strftime("%H:%M:%S"), "Exception:", e
+	    raise
+
+        # done scanning ftp
+        print time.strftime("%H:%M:%S"), "calling dbthread shutdown"
+        self.dbthread.shutdown()
+	while (self.dbthread.isAlive()):
+	    print time.strftime("%H:%M:%S"), "waiting on db thread"
+	    try:
+		time.sleep(5)
+	    except Exception, e:
+		print time.strftime("%H:%M:%S"), "Exception:",e
+		self.dbthread.die()
+	
+	print time.strftime("%H:%M:%S"), "joining dbthread"
+    	self.dbthread.join(timeout=5)
+	print time.strftime("%H:%M:%S"), "done"
+
+class ftplib_test:
+    def __init__(self, host):
+        self.host = host
+	self.dbq = Queue.Queue()
+        
+        self.dbthread = FsimDB(self.host, self.dbq)
+    
+        print time.strftime("%H:%M:%S"), "FsimCrawl: ftp instance"
+	self.ftp = FTP(host)
+        print time.strftime("%H:%M:%S"), "FsimCrawl: login"
+	self.ftp.login()
+    
+    def list_dir(self, path, dir):
+	assert dir
+        newpath = list(path)+list([dir])
+	lines = []
+	fulldir = '/'.join(newpath)
+	self.ftp.cwd(fulldir)
+        #self.ftp.retrlines("LIST %s" % fulldir, lines.append)
+	self.ftp.retrlines("LIST", lines.append)
+	if newpath==['/']:
+            newpath=[""]
+	return (newpath, lines)
+    
+    def crawl(self):
+	pathlines = []
+	toscan = []
+	pathlines.append(self.list_dir([], "/"))
+	c = 0
+	fc = 0
+	while (pathlines):
+	    if not self.dbthread.isAlive():
+                print time.strftime("%H:%M:%S"), "dbthread is dead ! exiting now!"
+		break
+		
+
+	    path, lines = pathlines.pop(0)
+	    for line in lines:
+		#print len(pathlines), path, line
+		try:
+        	    fullmode, links, owner, group, size, rest = line.split(None, 5)
+		except ValueError, e:
+		    if "total" in line: continue
+	
+		    print time.strftime("%H:%M:%S"), "FsimCrawl: ftp error:", e
+		    print time.strftime("%H:%M:%S"), "           in line:", line
+		    continue
+		
+		try:
+		    size = int(size)
+		except ValueError, e:
+		    # trying to include badly behaving ftp servers
+		    # that only have one field for ownership
+		    rest = size+" "+rest
+		    size = int(group)
+		    
+    		isdir = fullmode[0]=='d'
+    	    	islink = fullmode[0]=='l'
+	    	mode = fullmode[1:]
+    		date = rest[:12]
+		name = rest[13:]
+
+		if (c<256):
+		    c+=1
+        	else:
+        	    print time.strftime("%H:%M:%S"), "FsimCrawler:", len(pathlines)
+            	    c=0
+		
+		if (fc<2048):
+		    fc+=1
+		else:
+	    	    # make sure ftp doesn't timeout
+		    print time.strftime("%H:%M:%S"), "ftp anti-timeout"
+		    fc = 0
+		    self.ftp.cwd("/")
+	    
+		if name=="." or name=="..":
+		    continue
+	    
+        	if (isdir):
+		    self.dbq.put(["DIR", path, name, size])
+		    print time.strftime("%H:%M:%S"), "FsimCrawler scan: %s/%s" % ("/".join(path), name)
+		    try:
+		    	toscan.append(self.list_dir(path, name))
+		    except:
+		    	print "Dir List Error";
+        	elif (islink):
+		    # ignoring links FIXME later
+		    pass
+        	else:
+            	    # is file
+		    self.dbq.put(["FILE", path, name, size])
+	    #endfor
+	    for todo in toscan:
+		#print time.strftime("%H:%M:%S"), "FsimCrawler scan: %s/%s" % ("/".join(path), todo)
+		#pathlines.append(self.list_dir(path, todo))
+		pathlines.append(todo)
+	    toscan = []
+    
+    def run(self):
+        print time.strftime("%H:%M:%S"), "FsimCrawl: run"
+        # start test
+        self.dbthread.start()
+
+	try:
+            self.crawl()
+        except KeyboardInterrupt, err:
+	    self.dbthread.die()
+	    print time.strftime("%H:%M:%S"), "KeyboardInterrupt:", str(err)
+        except Exception, e:
+	    self.dbthread.die()
+	    print time.strftime("%H:%M:%S"), "Exception:", e
+	    raise
+
+        # done scanning ftp
+        print time.strftime("%H:%M:%S"), "calling dbthread shutdown"
+        self.dbthread.shutdown()
+	while (self.dbthread.isAlive()):
+	    print time.strftime("%H:%M:%S"), "waiting on db thread"
+	    try:
+		time.sleep(5)
+	    except Exception, e:
+		print time.strftime("%H:%M:%S"), "Exception:",e
+		self.dbthread.die()
+	
+	print time.strftime("%H:%M:%S"), "joining dbthread"
+    	self.dbthread.join(timeout=5)
+	print time.strftime("%H:%M:%S"), "done"
+
+if __name__=="__main__":
+    # feed ip's only at this time
+    if (len(sys.argv)!=3):
+        print "usage: %s [crawl][file] ip_address" % sys.argv[0]
+	sys.exit()
+    if sys.argv[1]=="crawl":
+	ftplib_test(str(sys.argv[2])).run()
+    elif sys.argv[1]=="file":
+	cpickle_test(str(sys.argv[2])).run()
+    else:
+	print "invalid command line."
+
+    

src/com/idca/fsim/core/FsimContext.java

@@ -0,0 +1,114 @@
+/*
+ * 
+ * 
+ */
+
+package	com.idca.fsim.core;
+
+import com.idca.fsim.core.jdbc.ConnectionFactory;
+import	com.idca.fsim.core.plugins.SearchSource;
+
+
+import	java.util.ArrayList;
+import	java.util.HashMap;
+import	java.util.Map;
+import	java.util.List;
+import	java.util.Iterator;
+import	java.util.Collection;
+import	java.util.EventListener;
+import	java.util.logging.Logger;
+
+import	java.sql.Connection;
+import	java.sql.SQLException;
+
+
+/**
+ * An FsimContext
+ * 
+ * Note: because we use here only the non-synchronized Collections 
+ * we must synchronize this objectes are stuctural changed (add/remove) !!
+ * 
+ * @author Willem Cazander
+ * @version 1.0 
+ */
+public class FsimContext
+{
+	/** */
+	private Logger logger = Logger.getLogger(FsimContext.class.getName());
+	private Map searchSources = new HashMap();
+	private List listeners = new ArrayList(1);
+	private String name = null;
+	
+	private Map jspIncludePages = new HashMap(3);	
+
+	// ============ jspIncludePages
+	
+	public void addJSPIncludePage(String name,String file)
+	{
+		synchronized(jspIncludePages)
+		{
+			jspIncludePages.put(name,file);
+		}
+		logger.info("Adding plugin page: "+name+" file: "+file);
+	}
+	public String getJSPIncludePage(String name)
+	{
+		return (String)jspIncludePages.get(name);
+	}
+	
+	// ============== the name of this search context.
+	
+	public void setName(String name)
+	{
+		this.name = name;
+	}
+	public String getName()
+	{
+		return name;
+	}
+	
+	
+	// ================ listeners
+	
+	public List getListeners()
+	{
+		return listeners;
+	}
+	
+	public void addListener(EventListener eventListener)
+	{
+		logger.info("Adding EventListener");
+		synchronized(listeners)
+		{
+			listeners.add(eventListener);
+		}
+	}
+	
+	
+	// ================= search sources
+	
+	public Collection getSearchSources()
+	{
+		return searchSources.values();
+	}
+	public SearchSource getSearchSource(String name)
+	{
+		return (SearchSource)searchSources.get(name);
+	}
+	public void addSearchSource(SearchSource searchSource)
+	{
+		logger.info("Adding dataSearchSource: "+searchSource.getName());
+		synchronized(searchSources)
+		{
+			searchSources.put(searchSource.getName(),searchSource);
+		}
+	}
+	
+	
+	// ================== the db functions.
+
+	public Connection getConnectionByName(String dbName) throws SQLException
+	{
+		return ConnectionFactory.getConnection(dbName);
+	}
+}

src/com/idca/fsim/core/FsimListener.java

@@ -0,0 +1,132 @@
+/*
+ * Created on jul 16, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package	com.idca.fsim.core;
+
+import com.idca.fsim.core.config.ConfigParser;
+
+import javax.servlet.ServletContext;
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+
+import java.io.InputStream;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+
+/**
+ * Starts fsim from a web context.
+ * @author Willem Cazander
+ * @version 1.0 28/06/2005
+ */
+public class FsimListener implements ServletContextListener 
+{
+	/** */
+	private Logger logger = Logger.getLogger(FsimListener.class.getName());
+	
+	/**
+	 * This method is called when the servlet context is<br>
+	 * initialized(when the Web application is deployed). <br>
+	 * You can initialize servlet context related data here.<br>
+	 */
+	public void contextInitialized(ServletContextEvent event)
+	{
+		initLogging();
+		
+		ServletContext sc = event.getServletContext();
+		String file = sc.getRealPath(sc.getInitParameter("FSIM_CONFIG_FILE"));
+		if(file==null)
+		{
+			logger.warning("No config file given!! giving up, set parameter: "+"FSIM_CONFIG_FILE");
+		//	return;
+		}
+		try {
+			startFsim(file);
+			
+		} catch (Exception e) {
+			logger.warning("Fsim could not be started: "+e.getMessage());
+			e.printStackTrace();
+		}
+	}
+	
+	private void initLogging()
+	{
+		
+    	try {
+    		InputStream loggingProperties = null;
+    		String dev = "false"; //ApplicationSettings.getSetting("DEVELOPMENT");
+    		
+    		if("true".equalsIgnoreCase(dev))
+    		{
+        		// 1st try to load the non-cvs properties
+        		try
+        		{
+        			loggingProperties = this.getClass().getResourceAsStream("/com/idca/fsim/resources/logging/logging-development.properties");
+        		} catch (Throwable overrideT) {
+        		}
+    		}	
+    		// else the default properties
+    		if(loggingProperties==null)
+    		{
+    			loggingProperties = this.getClass().getResourceAsStream("/com/idca/fsim/resources/logging/logging.properties");
+    			logger.info("Loaded default logging properties");
+    		} else {
+    			logger.info("Loaded development logging properties");
+    		}
+    		
+    		LogManager.getLogManager().readConfiguration( loggingProperties );
+    		loggingProperties.close();
+    	} catch (Throwable logT) {
+    		// PRINT AS STACKTRACE because when we fuck the logging there is no log message 
+    		logT.printStackTrace();
+    	}
+    	//reinit logger for new levels
+    	 logger = Logger.getLogger(this.getClass().getName());
+    	
+    	checkLogLevels();
+	}
+	
+	private void checkLogLevels()
+	{
+    	// TEST all log levels
+    	logger.info			("TESTING LOG LEVELS:\n");
+    	logger.finest		("LOGLEVEL ENABLED");
+    	logger.finer		("LOGLEVEL ENABLED");
+    	logger.fine			("LOGLEVEL ENABLED");
+    	logger.config		("LOGLEVEL ENABLED");
+    	logger.warning		("LOGLEVEL ENABLED");
+    	logger.info			("LOGLEVEL ENABLED\n");		
+	}
+		
+	
+	
+	
+	/**
+	 * This method is invoked when the Servlet Context.<br> 
+	 * (the Web application) is undeployed or <br>
+	 * the J2EE Server shuts down.<br>
+	 */
+	public void contextDestroyed(ServletContextEvent event)
+	{
+	}
+	
+	
+	private void startFsim(String fileName)
+	{
+		try {
+			FsimContext fc = ConfigParser.parse(fileName);
+			FsimServer.addFsimContext(fc);
+		} catch (Exception e) {
+			logger.warning("Error while loading: "+fileName+" message: "+e.getMessage());
+			e.printStackTrace();
+		}
+		
+		logger.info("done starting fsim");	
+	}
+	
+
+	
+}

src/com/idca/fsim/core/FsimServer.java

@@ -0,0 +1,60 @@
+/*
+ * 
+ * 
+ */
+
+package	com.idca.fsim.core;
+
+import com.idca.fsim.core.plugins.SearchSource;
+import	com.idca.fsim.core.plugins.listeners.FsimContextListener;
+import	com.idca.fsim.core.plugins.listeners.FsimContextEvent;
+
+import	java.util.Hashtable;
+import	java.util.Collection;
+import	java.util.Iterator;
+
+/**
+ * Manages the different Fsim contexts
+ * 
+ * @author Willem Cazander
+ * @version 1.0 20/07/2005
+ */
+public class FsimServer
+{
+	static private Hashtable contexts = new Hashtable(1);
+	
+	static public void addFsimContext(FsimContext fsimContext)
+	{
+		if(contexts.get(fsimContext.getName())!=null)
+		{
+			System.out.println("Can't add an FsimContext with an already loaded name: "+fsimContext.getName());
+			return;
+		}
+		contexts.put(fsimContext.getName(),fsimContext);
+		
+		// send events
+		System.out.println("sending events of "+fsimContext.getName());
+		Iterator i = fsimContext.getListeners().iterator();
+		while(i.hasNext())
+		{
+			Object object = i.next();
+			System.out.println("Object == "+object);
+			if(object instanceof FsimContextListener) 
+			{
+				FsimContextListener fcl = (FsimContextListener)object;
+				fcl.contextInitialized(new FsimContextEvent(fsimContext));
+			}
+		}
+	}
+	
+	static public FsimContext getFsimContext(String name) {
+		if(name==null) {
+			return null;
+		}
+		return (FsimContext)contexts.get(name);
+	}
+	
+	static public Collection getFsimContexts() {
+		return contexts.values();
+	}
+}

src/com/idca/fsim/core/LetterMap.java

@@ -0,0 +1,55 @@
+/*
+ * 
+ */
+
+package	com.idca.fsim.core;
+
+import	com.idca.fsim.models.Word;
+
+import	java.util.Collection;
+import	java.util.HashSet;
+import	java.util.Hashtable;
+import	java.util.Map;
+
+/**
+ * An LetterMAp
+ * 
+ * @author Willem Cazander
+ * @version 1.0 
+ */
+public class LetterMap
+{
+	/** */
+	private Map wordSets = new Hashtable(240);
+	
+	/**
+	 * 
+	 * @param ch
+	 * @param word
+	 */
+	public synchronized void addWord(Character ch,Word word)
+	{
+		Collection words = (Collection)wordSets.get(ch);
+		if(words==null) {
+			words = new HashSet();
+			wordSets.put(ch,words);
+		}
+		words.add(word);
+	}
+	
+	
+	/**
+	 * 
+	 * @param ch
+	 * @return
+	 */
+	public Collection getWords(Character ch)
+	{
+		return (Collection)wordSets.get(ch);
+	}
+	
+	protected Map getWordSet()
+	{
+		return wordSets;
+	}
+}

src/com/idca/fsim/core/Searcher.java

@@ -0,0 +1,114 @@
+/*
+ * Created on jul 16, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package	com.idca.fsim.core;
+
+import	com.idca.fsim.core.comparators.WordListComparator;
+import	com.idca.fsim.core.comparators.WordComparator;
+import	com.idca.fsim.models.*;
+
+import	java.util.logging.Logger;
+
+import	java.util.List;
+import	java.util.HashSet;
+import	java.util.Iterator;
+import	java.util.ArrayList;
+import	java.util.Collection;
+import	java.util.Collections;
+
+import javax.servlet.ServletException;
+
+/**
+ * Searches
+ * 
+ * @author Willem Cazander
+ * @version 1.0 Jan 22, 2006
+ */
+public class Searcher
+{
+	private WordLists wl = null;
+	private Logger logger = Logger.getLogger(Searcher.class.getName());
+	private ArrayList searchSources = new ArrayList(3);
+	
+	public Searcher(WordLists wl)
+	{
+		this.wl=wl;
+	}
+	
+	public Collection searchWords(String word,boolean inWord)
+	{
+		String[] keyWords = word.split(" "); 
+		
+		
+		
+		
+		//List result = null;
+		List last = new ArrayList(19);
+		for(int i=0;i<keyWords.length;i++)
+		{
+			String text = keyWords[i];
+			//System.out.println("Doing "+text);
+			List words = wl.getWords(text,inWord,false,false);
+			//System.out.println("Searcher Doing "+text+" got "+words.size());
+			Iterator a = words.iterator();
+			while(a.hasNext())
+			{
+			    Word w = (Word)a.next();
+			    //System.out.println("word = "+w.text);
+			}
+		
+			last.addAll(words);
+		}
+		logger.info("Searched for word: >> "+word+" << found "+last.size()+" results.");
+	
+		// sort on hist list size and Keywords
+		WordComparator wc =new WordComparator();
+		wc.setKeywords(keyWords);
+		Collections.sort(last,wc);
+		
+	//	Collections.sort(last,new WordClickedComperator());
+		return last;
+	}
+	
+	public Collection searchHits(String text,boolean inWord)
+	{
+		Collection last = searchWords(text,inWord);
+		
+		ArrayList hists = new ArrayList(100);
+		Iterator x = last.iterator();
+		while(x.hasNext()) {
+			Word word = (Word)x.next();
+			hists.addAll(word.hits);
+		}
+
+		return hists;
+	}
+	
+	public void setSearchSources(String searchSources)
+	{
+		addSearchSources(this.searchSources,searchSources);
+	}
+	
+	public boolean hasSearchSource(String searchSource)
+	{
+		if(searchSources.contains(searchSource)) {
+			return true;
+		}
+		return false;
+	}
+	
+	private void addSearchSources(List list,String sources)
+	{
+		String[] pathsA=sources.split(";");
+		for(int i=0;i<pathsA.length;i++)
+		{
+			String path = pathsA[i];
+			list.add(path);
+		}
+	}
+	
+}

src/com/idca/fsim/core/WordLists.java

@@ -0,0 +1,220 @@
+/*
+ * Created on jul 16, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package	com.idca.fsim.core;
+
+import com.idca.fsim.core.comparators.WordListComparator;
+import	com.idca.fsim.models.Word;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import	java.util.Map;
+import	java.util.Hashtable;
+import	java.util.List;
+import	java.util.HashSet;
+import	java.util.Collection;
+import	java.util.logging.Logger;
+
+/**
+ * An Words list
+ * 
+ * @author Willem Cazander
+ * @version 1.0 
+ */
+public class WordLists
+{	
+	private Map letters = new Hashtable(140);
+	private Logger logger = Logger.getLogger(WordLists.class.getName());
+	
+	/**
+	 * 
+	 * @param word
+	 */
+	public void put(Word word)
+	{
+		int wordLength = word.text.length();
+		for(int i=0;i<wordLength-1;i++)
+		{
+			Character ch1 = new Character(word.text.charAt(i));
+			Character ch2 = new Character(word.text.charAt(i+1));
+			
+			LetterMap letterMap = (LetterMap)letters.get(ch1);
+			if(letterMap==null) {
+				letterMap = new LetterMap();
+				synchronized(letters)
+				{
+					letters.put(ch1,letterMap);
+				}
+			}
+			synchronized(letterMap)
+			{
+				letterMap.addWord(ch2,word);
+			}
+		}
+	}
+	
+	/**
+	 * THis methode returns a Collection of all matching text
+	 * like: 'text' => te+ex+xt = returned. 
+	 * @param text
+	 * @return
+	 */
+	public Collection getWords(String text)
+	{
+		//logger.info("Starting search on "+text);
+		// no spaces should be in text
+		int textLength = text.length();
+		List allWords = new ArrayList(6);
+		for(int i=0;i<textLength-1;i++)
+		{
+			Character ch1 = new Character(text.charAt(i));			
+			Character ch2 = new Character(text.charAt(i+1));
+			
+			LetterMap letterMap = (LetterMap)letters.get(ch1);
+			Collection words = null;
+			if(letterMap!=null) {
+				words = letterMap.getWords(ch2);
+			}
+			
+			if(words==null) {
+				//System.err.println("letterpair could not be found: '"+ch1+ch2+"'");
+				continue;
+			}
+			// filter out double letterpairs 
+			if(!allWords.contains(words)) {
+				allWords.add(words);
+			}
+		}
+		// sort on list size
+		Collections.sort(allWords,new WordListComparator());
+		
+		Collection result = null;
+		Iterator i = allWords.iterator();
+		while(i.hasNext())
+		{
+			Collection list = (Collection)i.next();
+			if(result==null) {
+				//System.out.println("Starting with=null list="+list.size());
+				result = new HashSet(list);
+			} else {
+				//System.out.println("retrain with="+result.size()+" list="+list.size());
+				result.retainAll(list);
+			}
+			if(result.size()==0) {
+				// not contineu if we didn't found anything.
+				break;
+			}
+		}
+		if(result==null) {
+			result = new ArrayList(0);
+		}
+		
+		//logger.info("returning result size: "+result.size());
+		return result;
+	}
+	/**
+	 * 
+	 * @param text should not contain spaces !!
+	 * @param inWord if false word should startWith text
+	 * @param matchCase upper/lower case matching
+	 * @param exactMatch then text equals word.text
+	 * @return
+	 */
+	public List getWords(String text,boolean inWord,boolean matchCase,boolean exactMatch)
+	{
+		List result = new ArrayList(10);
+		Collection words = getWords(text);
+		Iterator i = words.iterator();
+		
+		if(!matchCase) {
+			text = text.toLowerCase();
+		}
+		
+		while(i.hasNext())
+		{
+			Word word = (Word)i.next();
+			
+			String wordText = word.text;
+			if(!matchCase) {
+				wordText = word.text.toLowerCase();
+			}
+			//System.out.println("=========search="+wordText+" list="+text);
+			
+			// think twice this is reversed code !!
+			boolean found = true;
+			int index = wordText.indexOf(text);
+			if(!inWord) {
+				// if is other the start pos them next word/
+				if(index!=0) {
+					found = false;
+				}
+			} else {
+				if(index==-1) {
+					found = false;
+				}
+			}
+			if(found && exactMatch && !wordText.equals(text)) {
+				found = false;
+			}
+			
+			if(found) {
+				result.add(word);
+			}
+		}
+		return result;
+	}
+	
+	
+	public Word getWord(String text)
+	{
+		List col = getWords(text,false,true,true);
+		if(col.size()==1) {
+			return (Word)col.get(0);
+		}
+		if(col.size()==0) {
+			// not found word.
+			return null;
+		}
+		logger.warning("list is bigger than one should not happen: collection size: "+col.size()+" word="+text);
+		
+		Iterator i = col.iterator();
+		while(i.hasNext()) {
+			
+			Word word = (Word)i.next();
+			System.out.println(" word == "+word.text);
+			
+		}
+		
+		return null;
+	}
+	
+	public LetterMap getLetterMap(Character ch)
+	{
+		return (LetterMap)letters.get(ch);
+	}	
+	
+	public String dumpInfo()
+	{
+		Iterator keys = letters.keySet().iterator();
+		while(keys.hasNext())
+		{
+			Object key1 = keys.next();
+			LetterMap letterMap =(LetterMap)letters.get(key1);
+			
+			Iterator letters2 = letterMap.getWordSet().keySet().iterator();
+			while(letters2.hasNext())
+			{
+				Object key2 = letters2.next();
+				Collection list2 = (Collection)letterMap.getWordSet().get(key2);
+				System.out.println("Letter: "+key1+key2+" words: "+list2.size());
+			}
+		}
+		return null;
+	}
+}

src/com/idca/fsim/core/comparators/HitComperator.java

@@ -0,0 +1,36 @@
+/*
+ * Created on jul 16, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package	com.idca.fsim.core.comparators;
+
+import	com.idca.fsim.models.Hit;
+import	java.util.Comparator;
+
+/**
+ * compares 2 Hits.
+ * 
+ * @author Willem Cazander
+ * @version 1.0 
+ */
+public class HitComperator implements Comparator
+{
+	
+	/**
+	 * 
+	 */
+	public int compare(Object o1,Object o2)
+	{
+		Hit hit1 = (Hit)o1;
+		Hit hit2 = (Hit)o2;
+		
+		if(hit1.id==hit2.id) 
+		{
+			
+		}
+		return 0;
+	}
+}

src/com/idca/fsim/core/comparators/WordClickedComparator.java

@@ -0,0 +1,39 @@
+/*
+ * Created on jul 16, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package com.idca.fsim.core.comparators;
+
+import	com.idca.fsim.models.*;
+import	java.util.*;
+
+/**
+ * Compare words for there click rate.
+ * 
+ * @author Willem Cazander
+ * @version 1.0 
+ */
+public class WordClickedComparator implements Comparator
+{
+	/**
+	 * 
+	 */
+	public int compare(Object object1,Object object2) throws ClassCastException
+	{
+		Word word1 = (Word)object1;
+		Word word2 = (Word)object2;
+		
+		if(word1.clicked>word2.clicked)
+		{
+			return 1;
+		}
+		if(word1.clicked<word2.clicked)
+		{
+			return -1;
+		}
+		return 0;
+	}
+}

src/com/idca/fsim/core/comparators/WordComparator.java

@@ -0,0 +1,66 @@
+/*
+ * Created on jul 16, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package com.idca.fsim.core.comparators;
+
+import	com.idca.fsim.models.*;
+import	java.util.*;
+
+/**
+ * Compare 2 Words.
+ * 
+ * @author Willem Cazander
+ * @version 1.0 
+ */
+public class WordComparator implements Comparator
+{
+    private String[] keyWords = null;
+    
+	/**
+	 * 
+	 */
+	public int compare(Object object1,Object object2) throws ClassCastException
+	{
+		Word word1 = (Word)object1;
+		Word word2 = (Word)object2;
+		
+		if(keyWords==null) {
+		    return 0;
+		}
+	
+		int w1b = 0;
+		int w2b = 0;
+
+		// check for keywords.
+		for(int i=0;i<keyWords.length;i++)
+		{
+			String text = keyWords[i].toLowerCase();
+			if(text.indexOf(word1.text.toLowerCase())!=-1) {
+			    w1b++;
+			}
+			if(text.indexOf(word2.text.toLowerCase())!=-1) {
+			    w2b++;
+			}
+		}	
+		System.out.println("w1b=="+word1.text+" :"+w1b+" w2b=="+word2.text+":"+w2b);
+		
+		if(w1b>w2b)
+		{
+			return -1;
+		}
+		if(w1b<w2b)
+		{
+			return 1;
+		}
+		return 0;
+	}
+
+	public void setKeywords(String[] keyWords)
+	{
+	    this.keyWords=keyWords;
+	}
+}

src/com/idca/fsim/core/comparators/WordListComparator.java

@@ -0,0 +1,33 @@
+/*
+ * Created on jul 16, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package	com.idca.fsim.core.comparators;
+
+import	java.util.*;
+
+/**
+ * Compares WordLists on its size.
+ * 
+ * @author Willem Cazander
+ * @version 1.0 
+ */
+public class WordListComparator implements Comparator
+{
+	/**
+	 * 
+	 */
+	public int compare(Object object1,Object object2) throws ClassCastException
+	{
+		Collection list1 = (Collection)object1;
+		Collection list2 = (Collection)object2;
+		
+		Integer size1 = new Integer(list1.size());
+		Integer size2 = new Integer(list2.size());
+	
+		return size1.compareTo(size2);
+	}
+}

src/com/idca/fsim/core/config/ConfigParser.java

@@ -0,0 +1,70 @@
+/*
+ * Created on jul 16, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package	com.idca.fsim.core.config;
+
+import	com.idca.fsim.core.FsimContext;
+
+import	com.mbuyu.foei.xml.XMLDebugTagHandler;
+import	com.mbuyu.foei.xml.XMLHandler;
+
+import	javax.xml.parsers.SAXParser;
+import	javax.xml.parsers.SAXParserFactory;
+import	javax.xml.parsers.ParserConfigurationException;
+import	org.xml.sax.SAXException;
+import	java.io.FileInputStream;
+import	java.io.InputStream;
+import	java.io.IOException;
+
+/**
+ * Parseres fsim config xml syntax from an source.
+ * 
+ * @author Willem Cazander
+ * @version 1.0 16/07/2005
+ */
+public class ConfigParser
+{
+	
+	/**
+	 * Opens this file and parse it xml fsim config syntaxS
+	 * @param fileName
+	 * @throws ParserConfigurationException
+	 * @throws SAXException
+	 * @throws IOException
+	 */
+	static public FsimContext parse(String fileName) throws ParserConfigurationException,SAXException,IOException
+	{
+		return parse(new FileInputStream(fileName));
+	}
+	
+
+	/**
+	 * Parse this xml stream as an fsim config syntax.
+	 * @param inputStream
+	 * @throws ParserConfigurationException
+	 * @throws SAXException
+	 * @throws IOException
+	 */
+	static public FsimContext parse(InputStream inputStream) throws ParserConfigurationException,SAXException,IOException
+	{
+		FsimContext fc = new FsimContext();
+		XMLHandler handler = new XMLHandler();
+		handler.addXMLHandler(new XMLDebugTagHandler());
+		handler.addXMLHandler(new FsimConfigTagHandler(fc));
+		//handler.addXMLHandler(new BundleTagHandler(fc));
+		handler.addXMLHandler(new JSPIncludePageTagHandler(fc));
+		//handler.addXMLHandler(new DataBaseTagHandler(fc));
+		handler.addXMLHandler(new SearchSourceTagHandler(fc));
+		handler.addXMLHandler(new ListenerTagHandler(fc));
+
+		SAXParserFactory factory = SAXParserFactory.newInstance();
+		factory.setNamespaceAware(true);
+		SAXParser saxParser = factory.newSAXParser();		       
+		saxParser.parse(inputStream,handler);
+		return fc;
+	}
+}

src/com/idca/fsim/core/config/FsimConfigTagHandler.java

@@ -0,0 +1,45 @@
+/*
+ * Created on jul 20, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package	com.idca.fsim.core.config;
+
+import	com.idca.fsim.core.FsimContext;
+import	com.mbuyu.foei.xml.AbstractXMLTagHandler;
+
+import	org.xml.sax.Attributes;
+import	org.xml.sax.SAXException;
+
+/**
+ * Handles the Tags of the config.
+ * 
+ * @author Willem Cazander
+ * @version 1.0 20/07/2005
+ */
+public class FsimConfigTagHandler extends AbstractXMLTagHandler
+{
+	/** */
+	private FsimContext fc = null;
+
+	public FsimConfigTagHandler(FsimContext fc)
+	{
+		this.fc=fc;
+		setTag("fsimConfig");
+	}
+	
+	/**
+	 * 
+	 */
+	public void startElement(String nameSpace,String tag,Attributes attributes) throws SAXException
+	{
+		if(getTag().equals(tag))
+		{
+			String name = attributes.getValue("name");
+			//System.out.println("Setting name: "+name);
+			fc.setName(name);
+		}
+	}
+}

src/com/idca/fsim/core/config/JSPIncludePageTagHandler.java

@@ -0,0 +1,45 @@
+/*
+ * Created on jul 23, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package	com.idca.fsim.core.config;
+
+import	com.idca.fsim.core.FsimContext;
+import	com.mbuyu.foei.xml.AbstractXMLTagHandler;
+
+import	org.xml.sax.Attributes;
+import	org.xml.sax.SAXException;
+
+/**
+ * The plugin pages
+ * 
+ * @author Willem Cazander
+ * @version 1.0 23/07/2005
+ */
+public class JSPIncludePageTagHandler extends AbstractXMLTagHandler
+{
+	/** */
+	private FsimContext fc = null;
+
+	public JSPIncludePageTagHandler(FsimContext fc)
+	{
+		this.fc=fc;
+		setTag("pluginPage");
+	}
+	
+	/**
+	 * 
+	 */
+	public void startElement(String nameSpace,String tag,Attributes attributes) throws SAXException
+	{
+		if(getTag().equals(tag))
+		{
+			String name = attributes.getValue("name");
+			String file = attributes.getValue("file");
+			fc.addJSPIncludePage(name,file);
+		}
+	}
+}

src/com/idca/fsim/core/config/ListenerTagHandler.java

@@ -0,0 +1,64 @@
+/*
+ * Created on jul 20, 2005
+ *
+ * Copyright 2004 IDCA. All rights reserved.
+ * IDCA PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package	com.idca.fsim.core.config;
+
+import	com.idca.fsim.core.FsimContext;
+import	com.mbuyu.foei.xml.AbstractXMLTagHandler;
+
+import	org.xml.sax.Attributes;
+import	org.xml.sax.SAXException;
+
+import	java.util.EventListener;
+import	java.util.logging.Logger;
+
+/**
+ * The Fsim Listeners
+ * 
+ * @author Willem Cazander
+ * @version 1.0 20/07/2005
+ */
+public class ListenerTagHandler extends AbstractXMLTagHandler
+{
+	/** */
+	private FsimContext fc = null;
+	/** */
+	private Logger logger = Logger.getLogger(this.getClass().getName());
+
+	public ListenerTagHandler(FsimContext fc)
+	{
+		this.fc=fc;
+		setTag("listener");
+	}
+	
+	/**
+	 * 
+	 */
+	public void startElement(String nameSpace,String tag,Attributes attributes) throws SAXException
+	{
+		if(getTag().equals(tag))
+		{
+			String className = attributes.getValue("class");
+
+			Object object = null;
+			Class objectClass = null;
+			try {	
+				objectClass = Class.forName(className);
+				object = objectClass.newInstance();				
+			} catch (ExceptionInInitializerError eie) {
+			} catch (LinkageError lae) {
+			} catch (InstantiationException ie) {
+			} catch (IllegalAccessException iae) {
+			} catch (ClassNotFoundException cnfe) {
+			}
+			System.out.println("adding listener: "+object);
+			if(object instanceof EventListener) {
+				fc.addListener((EventListener)object);
+			}
+		}
+	}
+}