Security By Encryption & Programatic Security

How can we add more security when we are writing a security program?( If we are not expert :) )

Think that your rival(attacker,intruder) is able to decrypt RSA or AES in an acceptable time.
Then we have to think on our flown,algorithms.
There are lots of things one can do. I will state some of them.

1) Adding garbage data
Poisson network, disk and database with random garbage data.

2)Partition files
Files are usually encrypted Symmetric. (Asymmetric is so slow for file encryption)
So at the last stage of flow, attacker is left with Symmetric decryption.

What we can do is partition files and encrypt each with Symmetric to
spend more time on file decryption.

3)File entropy
File entropy is randomness in a file.
This is useful in predicting patterns in a file.
One practice about this :

Steganography : concealing file in another file.

We were keeping chunked files in server which were all parts of o file.
We encrypted random parts with Asymmetric .
We made uniform entropy chunks .

*Define a target entropy.
*Calculate file entropy
*Add necessary (distribute over file) data to balance of each chunk.

Cv

Download Cv

ExecutorService Sample

For Thread Management one can use simple Executor Service. We had lots of small job but time consuming jobs which
could execute later. Below is the sample code.

If you want the jobs to be executed in the order you give , use sequential.
Example : You want to fire a register event and then an email notification.
You want these to be executed in the order you submitted to queue.

If you are not interested in the order just use pooled.

protected static ThreadLocal<reentrantlock> requestLock = new ThreadLocal<reentrantlock>( ) ;  
   protected static ThreadLocal<condition> requestCondition = new ThreadLocal<condition>( ) ;  
   public static ReentrantLock getLock( boolean isInit )  
   {  
     ReentrantLock lock = requestLock.get( ) ;  
     if( lock == null &amp;&amp; isInit )  
     {  
       lock = new ReentrantLock( true ) ;  
       requestLock.set( lock ) ;  
     }  
     return lock ;  
   }  
   public static Condition getCondition( boolean isInit )  
   {  
     Condition con = requestCondition.get( ) ;  
     if( con == null &amp;&amp; isInit )  
     {  
       con = getLock( true ).newCondition( ) ;  
       requestCondition.set( con ) ;  
     }  
     return con ;  
   }  
 protected static ExecutorService _sequentialExecutor = null ;  
   protected static ExecutorService _pooledExecutor = null ;  
   static  
   {  
     init( true ) ;  
   }  
   public static void init( boolean isForce )  
   {  
     if( _sequentialExecutor != null &amp;&amp; _pooledExecutor != null &amp;&amp; !isForce )  
     {  
       return ;  
     }  
     _sequentialExecutor = Executors.newSingleThreadExecutor( ) ;  
     _pooledExecutor = Executors.newFixedThreadPool( 4 ) ;  
   }  
   public static <t> Future<t> sequentialExec( YTask<t> task, boolean isWaitForRequest )  
   {  
     if( isWaitForRequest )  
     {  
       ReentrantLock lock = getLock( true ) ;  
       // initialize the condition as well  
       getCondition( true ) ;  
       task.setLock( lock ) ;  
       if( !lock.isHeldByCurrentThread( ) )  
       {  
         lock.lock( ) ;  
       }  
     }  
     return _sequentialExecutor.submit( ( Callable )task ) ;  
   }  
   public static <t> Future<t> sequentialExec( YScheduledTask task )  
   {  
     return _sequentialExecutor.submit( ( Callable )task ) ;  
   }  
   public static <t> Future<t> sequentialExec( Callable<t> callable )  
   {  
     return _sequentialExecutor.submit( callable ) ;  
   }  
   public static <t> Future<t> pooledExec( YTask<t> task, boolean isWaitForRequest )  
   {  
     if( isWaitForRequest )  
     {  
       ReentrantLock lock = getLock( true ) ;  
       // initialize the condition as well  
       getCondition( true ) ;  
       task.setLock( lock ) ;  
       if( !lock.isHeldByCurrentThread( ) )  
       {  
         lock.lock( ) ;  
       }  
     }  
     return _pooledExecutor.submit( ( Callable )task ) ;  
   }  

YTask is a utility class for callable services via this class.

import java.util.concurrent.Callable ;  
 import java.util.concurrent.TimeUnit ;  
 import java.util.concurrent.locks.Condition ;  
 import java.util.concurrent.locks.ReentrantLock ;  
 public class YTask<t> implements Callable<t>, Runnable  
 {  
   protected Callable<t> callable ;  
   protected long delayMs = 0 ;  
   protected ReentrantLock lock = null ;  
   protected Condition condition = null ;  
   public YTask( Callable<t> callable )  
   {  
     this.callable = callable ;  
   }  
   public Callable<t> getCallable( )  
   {  
     return this.callable ;  
   }  
   public long getDelayMs( )  
   {  
     return this.delayMs ;  
   }  
   public ReentrantLock getLock( )  
   {  
     return this.lock ;  
   }  
   public void setLock( ReentrantLock lock )  
   {  
     this.lock = lock ;  
   }  
   public Condition getCondition( )  
   {  
     return this.condition ;  
   }  
   public void setCondition( Condition condition )  
   {  
     this.condition = condition ;  
   }  
   @Override  
   public void run( )  
   {  
     try  
     {  
       call( ) ;  
     }  
     catch( Exception e )  
     {  
       e.printStackTrace( ) ;  
       throw new RuntimeException( "Error during task...", e ) ;  
     }  
   }  
   @Override  
   public T call( ) throws Exception  
   {  
     T res = null ;  
     if( this.delayMs > 0 )  
     {  
       try  
       {  
         Thread.sleep( this.delayMs ) ;  
       }  
       catch( Exception e )  
       {  
         e.printStackTrace( ) ;  
       }  
     }  
     if( this.lock != null )  
     {  
       if( !this.lock.isHeldByCurrentThread( ) )  
       {  
         this.lock.lock( ) ;  
       }  
       System.err.println( "LOCK Info ytask : isHeldByCurrentThread() : " + this.lock.isHeldByCurrentThread( ) + ", getHoldCount() : " + this.lock.getHoldCount( ) + " , " + this.lock.getQueueLength( ) ) ;  
     }  
     try  
     {  
       if( this.callable != null )  
       {  
         if( this.condition != null )  
         {  
           if( !this.condition.await( 180, TimeUnit.SECONDS ) )  
           {  
             sendErrorMessage( null, "Error during waiting for lock in YTask", "Error during waiting for lock in YTask" ) ;  
           }  
         }  
         res = this.callable.call( ) ;  
       }  
       return res ;  
     }  
     finally  
     {  
       if( this.lock != null )  
       {  
         this.lock.unlock( ) ;  
       }  
     }  
   }  
 }  

Running a local http server for dumping output to webpage

Running a local http server for dumping output to webpage.

At our project for encryption ,we had the constraint that we will not be
putting descrypted or open data to network.
There is a client program (Swing) running encrypt decrypt jobs.
Encrypted files are stored at server.

Also users will be able to download encrypted files into their local from a web page.
How could we do that?

1)We generated links like :
http://127.0.0.1:58997/download/-1584127920xe48f773bd522664b1f94d7c926e144bb

2)We opened an Httpserver listening the port specified.

3)User was seeing link in web page.But link was triggering our client program.

4)Program was downloading file parts from server,decrypting them
and writing to response of http request.

Below is piece of codes for this logic.It is not complete but enough.

import java.io.IOException ;
import java.net.InetSocketAddress ;
import java.util.HashMap ;
import java.util.Map ;

import com.sun.net.httpserver.HttpHandler ;
import com.sun.net.httpserver.HttpServer ;

public class KEmbeddedHttpServer 
{
    protected HttpServer server ;

    protected int port ;

    protected Map handlerMap = new HashMap<>( ) ;

    public KEmbeddedHttpServer( int port )
    {
        this.port = port ;
    }

    public HttpServer getServer( )
    {
        return this.server ;
    }

    public int getPort( )
    {
        return this.port ;
    }

    public void setPort( int port )
    {
        this.port = port ;
    }

    public void addHandler( String context, HttpHandler handler )
    {
        this.handlerMap.put( context, handler ) ;
    }

    public void start( ) throws IOException
    {
        this.server = HttpServer.create( new InetSocketAddress( this.port ), 0 ) ;

        for( String context : this.handlerMap.keySet( ) )
        {
            this.server.createContext( "/" + context, this.handlerMap.get( context ) ) ;
        }

        this.server.setExecutor( null ) ; // creates a default executor

        this.server.start( ) ;
    }
}



public void doHandleRequest( final HttpExchange httpExchange ) throws IOException
    {
        String requestPath = httpExchange.getRequestURI( ).getPath( ) ;
        
            try
            {
                

                final String fileName = decFileMeta.substring( 0, decFileMeta.indexOf( ", @" ) ).trim( ) ;

                final OutputStream os = httpExchange.getResponseBody( ) ;

                IKObserver observer = new IKObserver( )
                {
                    @Override
                    public Boolean onAction( String action, byte[ ] data )
                    {
                        try
                        {
                            if( BLOCK_READ.equals( action ) )
                            {
                                os.write( data ) ;
                            }
                            else if( START_OF_OPERATION.equals( action ) )
                            {
                                httpExchange.getResponseHeaders( ).add( "Content-Disposition", "attachment; filename=" + fileName ) ;
                                httpExchange.getResponseHeaders( ).add( "Transfer-encoding", "chunked" ) ;
                                httpExchange.sendResponseHeaders( 200, 0 ) ;
                            }
                            else if( END_OF_OPERATION.equals( action ) )
                            {
                                if( os != null )
                                {
                                    os.flush( ) ;
                                    os.close( ) ;
                                }
                            }

                            return Boolean.TRUE ;
                        }
                        catch( Exception ex )
                        {
                            ex.printStackTrace( ) ;
                            return false ;
                        }
                    }
                } ;


                
            }
            catch( Exception e )
            {
                e.printStackTrace( ) ;
                e.printStackTrace( new PrintWriter( this.writer ) ) ;
                httpExchange.getResponseHeaders( ).add( "Content-Disposition", "inline; filename=error.html" ) ;
                httpExchange.sendResponseHeaders( 500, this.baos.size( ) ) ;
            }

        

    }


 

How to write a zero knowledge server for encryption?

In our project for encryption we needed to make server as blind as possible.

We applied below algorithms for this purpose.

1) Server must not know which files it is storing.
Put files in parts.
Place them at random locations on server.
Put fake blocks.

2)Server file parts must be meaningless.
For attacking for basic files(raw text files) one can check occurring patterns for frequency analysis.
So zip files to decrease entropy.

3)While checking database attackers must not understand anything.
For example think you have a column as user id.

If attacker sees column id = 3 he will understand all files of user 3.
So every column must be kept encrypted.
But a hash or md5 of 3 will always be same .Ex : 344jkfjkdf994jjfjf
So an attacker can understand these files belong to same user.

So what is a good candidate for every row to have different values for same data?
Row id . Put row id into hashing or AES algorithm .

But still there is a pattern.
User Id + Row id
either post or prefix can give a clue.

So put time stamp at beginning.

Date.now + user id + row id

Every user must understand his files but not other people' files.
So AES this with user AES key


select CAST( AES_DECRYPT( UNHEX( enc_column), ? ) as char ) as user_info

FROM anytable_encyrpted HAVING user_info = concat( 'userid=(', ?, '),id=(', anytable_encyrpted_id , ')' ) " ;

? are parameters as follows :
User AES key, and user id

How to simulate a thumb driver in local

In a project we needed to simulate a thumb driver for multi user system ,
share public keys over network.


A thumb driver does not share it's private key.
We can simulate a thumb driver in our local.


Project will enable users of system to use their local laptops or
usb drives as thumb drive.

So let's write basic principles for this simulation.
1)All encryption decryption must occur at local machine.

2)All data send over network must be encrypted

3)User public keys must be kept at server.(Public key distribution server better to
be on a different server than data server)

4)RSA is slow for file encryption.Encrypt Files with AES.
Encrypt AES keys with every user's public key put to server.
***You have one door,one key.
for n users clone key and put in a box only that target user have key(user's private key)

Lucene Tester

There is no need to explain why someone needs Lucene .
There are also lots of samples over net. I am just putting our sample if anyone
encounters this link.

import java.io.File ;
import java.io.Reader ;
import java.io.Serializable ;
import java.io.StringReader ;
import java.util.HashMap ;
import java.util.List ;

import org.apache.lucene.analysis.Analyzer ;
import org.apache.lucene.analysis.standard.StandardAnalyzer ;
import org.apache.lucene.document.Document ;
import org.apache.lucene.document.Field ;
import org.apache.lucene.index.CorruptIndexException ;
import org.apache.lucene.index.IndexReader ;
import org.apache.lucene.index.IndexWriter ;
import org.apache.lucene.index.IndexWriter.MaxFieldLength ;
import org.apache.lucene.index.Term ;
import org.apache.lucene.queryParser.QueryParser ;
import org.apache.lucene.search.IndexSearcher ;
import org.apache.lucene.search.Query ;
import org.apache.lucene.search.ScoreDoc ;
import org.apache.lucene.search.Searcher ;
import org.apache.lucene.search.TermQuery ;
import org.apache.lucene.search.TopDocs ;
import org.apache.lucene.search.similar.MoreLikeThis ;
import org.apache.lucene.search.spell.Dictionary ;
import org.apache.lucene.search.spell.LuceneDictionary ;
import org.apache.lucene.search.spell.SpellChecker ;
import org.apache.lucene.store.Directory ;
import org.apache.lucene.store.FSDirectory ;
import org.apache.lucene.util.Version ;

public class YLuceneTester
{


    private final String indexDir = "D:\\indexDir" ;

    private final String spellDirPath = "D:\\spellDir" ;

    /**
     * create index
     */
    public boolean createIndex( ) throws Exception
    {
        //        if( true == ifIndexExist( ) )
        //        {
        //            return true ;
        //        }
        //        File dir = new File(dataDir);
        //        if(!dir.exists()){
        //            return false;
        //        }

        //File[] htmls = dir.listFiles();

        Directory fsDirectory = FSDirectory.open( new File( this.indexDir ) ) ;
        Analyzer analyzer = new StandardAnalyzer( Version.LUCENE_33 ) ;
        IndexWriter indexWriter = new IndexWriter( fsDirectory, analyzer, true, MaxFieldLength.UNLIMITED ) ;

        addDocument( indexWriter ) ;

        indexWriter.optimize( ) ;
        indexWriter.close( ) ;

        IndexReader indexReader = null ;
        try
        {
            indexReader = IndexReader.open( fsDirectory ) ;
            Dictionary dictionary = new LuceneDictionary( indexReader, "trans" ) ;
            FSDirectory spellDir = FSDirectory.open( new File( this.spellDirPath ) ) ;
            SpellChecker spellChecker = new SpellChecker( spellDir ) ;
            spellChecker.indexDictionary( dictionary ) ;
            spellChecker.close( );
        }
        finally
        {
            if( indexReader != null )
            {
                indexReader.close( ) ;
            }
        }
        return true ;

    }

    /**
     * Add one document to the Lucene index
     * @throws Exception 
     * @throws CorruptIndexException 
     */
    public void addDocumentDB( IndexWriter indexWriter ) throws CorruptIndexException, Exception
    {
        YOrganization org = YOrganization.getTopLevelOrganization( "TXG100" ) ;

        String hql = " Select y.trans,y.id FROM YEntityTranslation y where y.organization.id =  " + org.getId( ) ;

        HashMap parameters = new HashMap( ) ;

        List existingCatalogData = ( List )HibernateUtils.execHQL( hql, parameters, 0, 1000 ) ;          for( Object[ ] datas : existingCatalogData )         {             String trans = ( String )datas[ 0 ] ;             if( YClientUtils.isBlankTrim( trans ) )                 continue ;              System.err.println( trans ) ;             Document document = new Document( ) ;             //document.add( new Field( "path", path, Field.Store.YES, Field.Index.NO ) ) ;             document.add( new Field( "trans", trans, Field.Store.YES, Field.Index.ANALYZED ) ) ;              indexWriter.addDocument( document ) ;         }      }          public void addDocument( IndexWriter indexWriter ) throws CorruptIndexException, Exception     {         String[] items = new String[]{"African lion","African wild cat","African wild dog","dog","cat","lion"};                  for( String item : items )         {              Document document = new Document( ) ;             document.add( new Field( "trans", item, Field.Store.YES, Field.Index.ANALYZED ) ) ;              indexWriter.addDocument( document ) ;         }      }      public Query suggest( String queryString ,int distance) throws Exception     {         try         {             Directory fsDirectory = FSDirectory.open( new File( this.spellDirPath ) ) ;             SpellChecker spellChecker = new SpellChecker( fsDirectory ) ;             if( spellChecker.exist( queryString ) )             {                 return null ;             }             String[ ] similarWords = spellChecker.suggestSimilar( queryString, distance ) ;             if( similarWords.length == 0 )             {                 return null ;             }              System.err.println( " Term = " + queryString + " Suggestions :" ) ;             for( String similarWord : similarWords )             {                 System.err.println( " ) " + similarWord ) ;             }              return new TermQuery( new Term( "trans", similarWords[ 0 ] ) ) ;         }         catch( Exception e )         {             throw new Exception( e.getMessage( ) ) ;         }     }      public void searchIndex( String[ ] queryStrings ) throws Exception     {         Searcher searcher = new IndexSearcher( FSDirectory.open( new File( this.indexDir ) ) ) ;         QueryParser parser = new QueryParser( Version.LUCENE_CURRENT, "trans", new StandardAnalyzer( Version.LUCENE_CURRENT ) ) ;         for( String queryString : queryStrings )         {             System.out.println( "nsearching for: " + queryString ) ;             Query query = parser.parse( queryString ) ;             TopDocs results = searcher.search( query, 10 ) ;             System.out.println( "total hits: " + results.totalHits ) ;             ScoreDoc[ ] hits = results.scoreDocs ;             for( ScoreDoc hit : hits )             {                 Document doc = searcher.doc( hit.doc ) ;                 System.out.printf( "%5.3f %sn \n", hit.score, doc.get( "trans" ) ) ;             }         }         searcher.close( ) ;     }       /**      * judge if the index exists already      */     public boolean ifIndexExist( )     {         File directory = new File( this.indexDir ) ;         if( 0 < directory.listFiles( ).length )
        {
            return true ;
        }
        else
        {
            return false ;
        }
    }

    public String getIndexDir( )
    {
        return this.indexDir ;
    }


    public Query parse( String queryString ) throws Exception
    {
        QueryParser queryParser = new QueryParser( Version.LUCENE_CURRENT, "trans", new StandardAnalyzer( Version.LUCENE_CURRENT ) ) ;
        queryParser.setDefaultOperator( QueryParser.AND_OPERATOR ) ;
        return queryParser.parse( queryString ) ;
    }

    public void search( String queryString ,int distance ) throws Exception
    {
        long startTime = System.currentTimeMillis( ) ;
        IndexSearcher is = null ;
        FSDirectory spellDir = FSDirectory.open( new File( this.spellDirPath ) ) ;
        Directory fsDirectory = FSDirectory.open( new File( this.indexDir ) ) ;

        int minimumHits = 100 ;
        int minimumScore = 5 ;

        try
        {
            is = new IndexSearcher( fsDirectory ) ;
            Query query = parse( queryString ) ;

            TopDocs tdocs = is.search( query, 100 ) ;

            //Hits hits = is.search( query ) ;

            //            for( ScoreDoc sdoc :  tdocs.scoreDocs )
            //            {
            //                sdoc.
            //            }

            String suggestedQueryString = null ;
            if( tdocs.totalHits < minimumHits || tdocs.getMaxScore( ) < minimumScore )
            {
                Query didYouMean = suggest( queryString ,distance) ;
                if( didYouMean != null )
                {
                    suggestedQueryString = didYouMean.toString( "trans" ) ;
                }
            }

            long endTime = System.currentTimeMillis( ) ;

            //return new SearchResult( extractHits( hits ), hits.length( ), endTime - startTime, queryString, suggestedQueryString ) ;
        }
        finally
        {
            if( is != null )
            {
                is.close( ) ;
            }
        }
    }

    public void moreLikeThis( String text ) throws Exception
    {
        Directory fsDirectory = FSDirectory.open( new File( this.indexDir ) ) ;

        IndexReader indexReader = IndexReader.open( fsDirectory ) ;

        //        FuzzyLikeThisQuer flt = new FuzzyLikeThisQuery( 50, new StandardAnalyzer( ) ) ;
        //        flt.addTerms( "product critical update", "title", 0.75f, FuzzyQuery.defaultPrefixLength ) ;
        //        BooleanQuery q = ( BooleanQuery )flt.rewrite( r ) ;
        //        int minNumClauseMatches = Math.round( q.clauses( ).size( ) * 0.5f ) ;
        //        q.setMinimumNumberShouldMatch( minNumClauseMatches ) ;

        IndexSearcher is = new IndexSearcher( FSDirectory.open( new File( this.indexDir ) ) ) ;

        MoreLikeThis mlt = new MoreLikeThis( indexReader ) ;
        mlt.setFieldNames( new String[ ] { "trans" } ) ;

        mlt.setMinWordLen( 2 ) ;
        mlt.setBoost( true ) ;

        Reader reader = new StringReader( text ) ;

        //Create the query that we can then use to search the index
        Query query = mlt.like( reader ) ;

        //Search the index using the query and get the top 5 results
        TopDocs topDocs = is.search( query, 5 ) ;

        //Create an array to hold the quotes we are going to
        //pass back to the client

        for( ScoreDoc scoreDoc : topDocs.scoreDocs )
        {
            //This retrieves the actual Document from the index using
            //the document number. (scoreDoc.doc is an int that is the

            System.err.print( "--" + scoreDoc.toString( ) ) ;
        }
        
        is.close( );

    }
    
    public static void init()
    {
        YLuceneTester luceneTester = new YLuceneTester( ) ;
        try
        {
            luceneTester.createIndex( ) ;

            //luceneTester.searchIndex( new String[ ] { "Cleaner" } ) ;
            //TermQuery q = ( TermQuery )luceneTester.suggest( "Claner" ) ;
            //q.extractTerms( terms )

            //luceneTester.moreLikeThis( "Clean" ) ;

            //            luceneTester.search( "Cleaner" ) ;
            //            luceneTester.search( "Cordless " ) ;
            //
            //            luceneTester.suggest( "Cleane" ) ;
            //            luceneTester.suggest( "Clean" ) ;
            //            luceneTester.suggest( "Clnr" ) ;

        }
        catch( Exception e )
        {
            e.printStackTrace( ) ;
        }
    }
    
    public static void tests()
    {
        YLuceneTester luceneTester = new YLuceneTester( ) ;
        try
        {

//            luceneTester.searchIndex( new String[ ] { "Afri" } ) ;
//            luceneTester.searchIndex( new String[ ] { "African" } ) ;
//            luceneTester.searchIndex( new String[ ] { "Africax" } ) ;
//            TermQuery q = ( TermQuery )luceneTester.suggest( "Claner" ) ;
//            q.extractTerms( terms )

            //luceneTester.moreLikeThis( "dog" ) ;

                        luceneTester.search( "Afrieen" ,1 ) ;
                        luceneTester.search( "Afrieen" ,1 ) ;
                        luceneTester.search( "Afrieen" ,1 ) ;
            //            luceneTester.search( "Cordless " ) ;
            //
                        luceneTester.suggest( "Afrieen",2 ) ;
                        luceneTester.suggest( "lion" ,2) ;
            //            luceneTester.suggest( "Clnr" ) ;

        }
        catch( Exception e )
        {
            e.printStackTrace( ) ;
        }
    }

    public static void main( String[ ] args )
    {
        //HibernateUtils._configFileName = "hibernate.hqltest.xml" ;
        
        //init( );
        
        tests( );



    }
}