package uk.ac.man.cs.lethe.internal.application.statistics

import java.io.{ File, PrintStream, FileOutputStream }
import java.io.FileNotFoundException
import java.io.PrintWriter
import java.text.SimpleDateFormat

import scala.collection.SortedSet
import scala.collection.mutable.HashMap
import scala.io.Source
import scala.sys.process._


object ExtendedStatistics { 

  val HELP = """

Usage: 

ExtendedStatistics [EXPERIMENT_DIR] [--byOntology] [--befDisjEl] [--graphs]

Generates graphs and csv-file based on files in current folder


"""

  val ONTOLOGY_INFORMATION_PATH = "."
  val CSV_FILE = "details.csv"


  var beforeDisjElimination = false

  def main(_args: Array[String]) = { 
    var args = _args
    var byOntology = false
    var bGraphs = false
    var bOneLine = false

    if(args.contains("--byOntology")){ 
      byOntology = true
      args=args.filterNot(_=="--byOntology")
    }
    
    if(args.contains("--befDisjEl")){ 
      beforeDisjElimination = true
      args = args.filterNot(_=="--befDisjEl")
    }

    if(args.contains("--graphs")){ 
      bGraphs = true
      args = args.filterNot(_=="--graphs")
    }

    if(args.contains("--oneLine")){ 
      bOneLine = true
      args = args.filterNot(_=="--oneLine")
    }
      


    val resultsFile: File = args match { 
      case Array() => new File("results.txt")
      case Array(dir) if (new File(dir)).exists && (new File(dir)).isDirectory => 
	new File(dir, "results.txt")
      case Array(file) if (new File(file)).exists => new File(file)
      case _ => { 
	println(HELP)
	System.exit(1)
	new File("")
      }
    }



    // Filtering data (contains families and sizes and stuff)
//    ExperimentFilter.loadInformation(new File(ONTOLOGY_INFORMATION_PATH))

    if(!bOneLine)
      println("\\section{"+resultsFile.getName.replaceAllLiterally("_","\\_")+"}")


    generateCSVFile(resultsFile)
    Statistics.statistics(CSV_FILE, byOntology=byOntology, nominals = !beforeDisjElimination, oneLine=bOneLine)
   
    if(bGraphs)
      graphs(resultsFile)
  }

  def graphs(resultsFile: File) = { 
    val details = createDetails(resultsFile)
    val prefix = ""//resultsFile.getName.replaceAllLiterally(".","_")
    generatePlot(details.sizes, "Input Size", "Output Size", prefix + "_sizes.ps")
    generateDurationsGraph2(details.durations, prefix + "_durations2.ps")
    println()
    generatePlot(details.moduleSizeOutputSize, "Module Size", "Uniform Interpolant Size", prefix + "_moduleSizes.ps")
    generatePlot(details.symbolsToForgetDuration, 
		 "Number of Forgotten Symbols", 
		 "Duration (s)",
		 prefix+"_symbolsToForget.ps",
		 orientationLine = false)
    // generateDurationsGraph2(details.runs.filter(_.completed).toSeq.map(_.definersLeft.toDouble), 
    // 			    prefix+"_definer.ps",
    // 			    yLabel="Num. of Definers")
    // generateDurationsGraph2(details.numberRestrictions.map(_.toDouble), 
    // 			    prefix+"_numRestrictions.ps",
    // 			    yLabel="Num. of Number Restrictions")


//    sizesGraph(resultsFile)
//    durationsGraph(resultsFile)
//    durationsGraph2(resultsFile)
//    println()
//    moduleSizeGraph(resultsFile)
//    symbolsToForgetGraph(resultsFile)
//    definerGraph(resultsFile)
//    numberRestrictionsGraph(prefix=prefix, resultsFile)
    println()
  }

  def sizesGraph(resultsFile: File, filterString: String="") = 
    sizesGraphN(resultsFile, -1, filterString)

  def sizesGraphN(resultsFile: File, maxExperiment: Int, filterString: String="") = { 

    val details = if(filterString!=""){ 
      val filter = new MappedFilter(filterString)
      createDetails(resultsFile, filter=filter)
    } else 
      createDetails(resultsFile, maxExperiment=maxExperiment)
    
    generatePlot(details.sizes, "Input Size", "Output Size", resultsFile.getName.replaceAllLiterally(".","_")+"_sizes.ps")
  }

  def durationsGraph(resultsFile: File, filterString: String="") = 
    durationsGraphN(resultsFile, -1,  filterString)

  def durationsGraphN(resultsFile: File, maxExperiment: Int, filterString: String="") = { 

    val details = if(filterString!=""){ 
      val filter = new MappedFilter(filterString)
      createDetails(resultsFile, filter=filter)
    } else 
      createDetails(resultsFile, maxExperiment=maxExperiment)
    
    generateDurationsGraph(details.durations, "durations.ps")
  }

  def durationsGraph2(resultsFile: File, filterString: String="") = 
    durationsGraph2N(resultsFile, -1,  filterString)

  def durationsGraph2N(resultsFile: File, maxExperiment: Int, filterString: String="") = { 

    val details = if(filterString!=""){ 
      val filter = new MappedFilter(filterString)
      createDetails(resultsFile, filter=filter)
    } else 
      createDetails(resultsFile, maxExperiment=maxExperiment)
    
    generateDurationsGraph2(details.durations, resultsFile.getName.replaceAllLiterally(".","_")+"_durations2.ps")
  }

  def numberRestrictionsGraph(prefix: String="", resultsFile: File) = { 
    val details = createDetails(resultsFile)
    
    generateDurationsGraph2(details.numberRestrictions.map(_.toDouble), 
			    prefix+"_numRestrictions.ps",
			    yLabel="Num. of Number Restrictions")
  }

  def definerGraph(resultsFile: File) = { 
    val details = createDetails(resultsFile)
    
    generateDurationsGraph2(details.runs.filter(_.completed).toSeq.map(_.definersLeft.toDouble), 
			    resultsFile.getName.replaceAllLiterally(".","_")+"_definer.ps",
			    yLabel="Num. of Definers")
  }

  def moduleSizeGraph(resultsFile: File, filterString: String="") = 
    moduleSizeGraphN(resultsFile, -1, filterString)

  def moduleSizeGraphN(resultsFile: File, maxExperiment: Int, filterString: String="") = { 

    val details = if(filterString!=""){ 
      val filter = new MappedFilter(filterString)
      createDetails(resultsFile, filter=filter)
    } else 
      createDetails(resultsFile, maxExperiment=maxExperiment)

    generatePlot(details.moduleSizeOutputSize, "Module Size", "Uniform Interpolant Size", resultsFile.getName.replaceAllLiterally(".","_")+"_moduleSizes.ps")
  }

  def symbolsToForgetGraph(resultsFile: File, filterString: String="") = 
    symbolsToForgetGraphN(resultsFile, -1, filterString)

  def symbolsToForgetGraphN(resultsFile: File, maxExperiment: Int, filterString: String="") = { 

    val details = if(filterString!=""){ 
      val filter = new MappedFilter(filterString)
      createDetails(resultsFile, filter=filter)
    } else 
      createDetails(resultsFile, maxExperiment=maxExperiment)
    generatePlot(details.symbolsToForgetDuration, 
		 "Number of Forgotten Symbols", 
		 "Duration (s)",
		 resultsFile.getName.replaceAllLiterally(".","_")+"_symbolsToForget.ps",
		 orientationLine = false)

  }



  def generateCSVFile(resultsFile: File) = { 

    val details = createDetails(resultsFile)

    val headLine = Seq("Ontology",
		       "Ontology-size",
		       "Ontology-Axioms",
		       "Ontology-Max-Axiom-Size",
		       "Family",
		       "Acyclic?",
		       "Expansion-Depth",
		       "Signature-Size",
		       "Completed?",
		       "Definers",
		       "Module-Size",
		       "To-Forget",
		       "Duration",
		       "Output-Size",
		       "Output-Axioms",
		       "Output-Max-Axiom-Size",
		       "Number-Restrictions",
		       "Output-ABox-Size",
		       "Output-ABox-Assertions",
		       "Disjunctive-Concept-Assertions",
		       "Nominals",
		       "Disjunctive-Eliminated?"
		       ).mkString("\t")

    val lines = details.runs.toSeq.map{ run => 
      val ontInf = ExperimentFilter.ontologyInformation(run.ontology)
      Seq(ontInf.name, 
	  run.inputSize,
	  run.axiomsInput,
	  run.maxAxiomSizeInput,
	  ontInf.family,
	  ontInf.acyclic,
	  ontInf.depth,
	  run.signatureSize, 
	  run.completed, 
	  run.definersLeft,
	  run.moduleSize,
	  run.symbolsToForget,
	  run.finishTime-run.startTime,
	  run.outputSize,
	  run.outputAxioms,
	  run.maxAxiomSizeOutput,
	  run.numberRestrictions,
	  run.outputABoxSize,
	  run.outputABoxAssertions,
	  run.disjunctiveAssertions,
	  run.nominals,
	  run.disjunctionsEliminated).map(_.toString).mkString("\t")
    }
    val string = headLine+"\n"+lines.mkString("\n")
//    Some(new PrintWriter(CSV_FILE)).foreach{p => p.write(string); p.close}
    new PrintStream(new FileOutputStream(new File(CSV_FILE), true)).println(string) // append
//    Path(CSV_FILE).toFile.write()
  }

  def createDetails(resultsFile: File,
		    experimentName: String = "Local Experiment",
		    maxExperiment: Int = -1,
		    filter: ExperimentFilter = EmptyFilter): ExperimentDetails = { 

//    val experimentDir = experimentPath(experimentName)

    val result = new ExperimentDetails(experimentName)

    val resultsLines = try{ 
//      Source.fromFile(new File(experimentDir, "results.txt"))(io.Codec("ISO-8859-1")).getLines()
      Source.fromFile(resultsFile)(io.Codec("ISO-8859-1")).getLines()
    } catch { 
      case _: FileNotFoundException => Seq[String]()
    }

    val reRedoing = """(Redoing:)""".r
    val reExperimentNumber = """Experiment number: (\d+)""".r
    val reOntologyName = """Ontology: (.+)""".r
    val reSignatureSize = """(?:Signature size|Words to forget): (\d+)""".r
    val reSignature = """(?:Desired signature:|Forgetting the following words:) Set\(([^\)]*)\)""".r
    val reNotCompleted = """.*(Execution has not been completed\.|Timeout exceeded\.)""".r
    val reDefinersLeft = """Cyclic definers left: (\d*)""".r
    val reModuleSize = """Size of module: (\d+)""".r
    val reSymbolsToForget = """Forgetting (\d+) symbols.""".r

    val reNumberRestrictions = """Number Restrictions in result: (\d+)""".r

    val reOutputABoxSize = """Output ABox size: (\d+)""".r
    val reOutputABoxAssertions = """Output ABox assertions: (\d+)""".r
    val reDisjunctiveAssertions = """Disjunctive Concept Assertions: (\d+)""".r
    val reNominals = """Nominals: (\d+)""".r
    val reDisjunctionsEliminated = """Eliminated definers.""".r // (sic!)
    val reEliminatingDisjunctions = """Eliminating disjunctions \(([^)]*)\)""".r 

    val reStartTime = """Started at (.*)""".r//\w\w\w \w\w\w \d\d \d\d:\d\d:\d\d \w\w\w \d\d\d\d)""".r
    val reFinishTime = """Finished at (.*)""".r//\w\w\w \w\w\w \d\d \d\d:\d\d:\d\d \w\w\w \d\d\d\d)""".r

    val reInputSize = """Size: (\d+)""".r
    val reOutputSize = """Size of the output \(split conjunctions\): (\d+)""".r
    val reOutputAxioms = """Number of axioms \(split conjunctions\): (\d+)""".r
    //val reOutputSize = """Size of the output: (\d+)""".r
    //val reOutputAxioms = """Number of axioms: (\d+)""".r

    val reAxiomsInput = """Axioms: (\d+)""".r
    val reMaxAxiomSizeInput = """Max axiom size: (\d+)""".r
					           
    val reMaxAxiomSizeOutput = """Max axiom size \(split conjunctions\): (\d+)""".r
//    val reMaxAxiomSizeOutput = """Max axiom size: (\d+)""".r

    val reSkippedSucc = """(Skipped \(succeeding signatures Timeout\).*)""".r

    val reStarted = """.*\|=.*""".r

    //Example: Wed May 22 20:57:04 BST 2013
    val dateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss zz yyyy")

//    var ontologiesProcessed = Set[String]()

    var inputSize = 0L
    var firstStart = 0L
    var lastStart = 0L
    var lastFinish = 0L
    var finished = true
    var currentExperiment = -1
    var moduleSize = 0
    var symbolsToForget = 0

    var lastOntology = ""

    var currentRun = SingleRun()

//    var runs: Set[SingleRun] = Set[SingleRun]()
    var runSeq = Seq[SingleRun]()

    var x=false

    var sigs = List(5,10,50,150)
    var lastSig = 0

    var last = ""
    val lines = resultsLines.foreach{ line =>

      var lineX = line
      if(last!=""){ 
       lineX = last+lineX
       last = ""
      }

      lineX match { 
	case reStarted() => currentRun.started = true
	case _ =>  ; // rest done in next match block
      }

      lineX match { 
	case reExperimentNumber(number) => currentRun = SingleRun(experimentNumber=number.toInt);
	case reRedoing(_)               => currentRun = SingleRun()
	case reOntologyName(name)       => currentRun.ontology = name; lastOntology=name
	case reSignature(sig)           => currentRun.signature = sig.split(", ").toSet
	case reStartTime(time)          => currentRun.startTime = dateFormat.parse(time).getTime
	case reInputSize(size)          => currentRun.inputSize = size.toInt
	case reModuleSize(size)         => currentRun.moduleSize = size.toInt
	case reSymbolsToForget(nr)      => currentRun.symbolsToForget = nr.toInt
	case reDefinersLeft(d)          => currentRun.definersLeft = d.toInt
	case reFinishTime(time)         => { 
	  currentRun.finished = true 
	  currentRun.finishTime = dateFormat.parse(time).getTime
	}
	case reOutputSize(size)         => currentRun.outputSize = size.toInt
	case reOutputAxioms(axioms)     => currentRun.outputAxioms = axioms.toInt
	case reSignatureSize(number)    => currentRun.signatureSize = number.toInt; lastSig=number.toInt
	case reNumberRestrictions(num)  => currentRun.numberRestrictions = num.toInt
	case reOutputABoxSize(size)     => currentRun.outputABoxSize = size.toInt
	case reOutputABoxAssertions(n)  => currentRun.outputABoxAssertions = n.toInt
	case reDisjunctiveAssertions(n) => currentRun.disjunctiveAssertions = n.toInt
	case reNominals(n)              => currentRun.nominals = n.toInt
	case reDisjunctionsEliminated()   => { 
	  currentRun.disjunctionsEliminated = true
	  if(beforeDisjElimination){ 
	    currentRun.finished = true
	    currentRun.completed = true
	  }
	}
	case reEliminatingDisjunctions(time) =>  { 
	  currentRun.finishTime = dateFormat.parse(time).getTime
	}
	case reAxiomsInput(number)      => currentRun.axiomsInput = number.toInt
	case reMaxAxiomSizeInput(number)=> currentRun.maxAxiomSizeInput = number.toInt
	case reMaxAxiomSizeOutput(num)  => currentRun.maxAxiomSizeOutput = num.toInt
	case reNotCompleted(_)          => { 
	  if(!currentRun.finished)
	    currentRun.completed = false 
	}
	// case reSkippedSucc(_) => { 
	//   val skippedRun = new SingleRun()
	//   skippedRun.ontology = lastOntology
	//   skippedRun.signatureSize = sigs(sigs.lastIndexOf(lastSig)+1)
	//   skippedRun.completed = false
	//   runSeq ++= Seq(skippedRun)
	//   runSeq ++= Seq(skippedRun)
	//   lastSig=skippedRun.signatureSize
	// }
	case _ => 
      }


//      if(currentRun.finishTime != -1){
      if(currentRun.finished && currentRun.startTime != -1L) { 
	// if(currentRun.outputSize == -1)
	//   currentRun.completed=false

	if((maxExperiment == -1 || currentRun.experimentNumber<maxExperiment) 
	   && filter.accepts(currentRun) 
	   && currentRun.started
//	   && currentRun.moduleSize!= -1
	 ){ 
	     // runs.find{ run => run.ontology==currentRun.ontology && 
	     // 	       run.signature==currentRun.signature } match { 
	     // 	case Some(run) => println("Doppelt: "+run)//runs -= run
	     // 	case None => ;
	     // }

	    runSeq ++= Seq(currentRun)
	}
	currentRun = SingleRun()
      }
	

//      assert(beforeDisjElimination || !(currentRun.finished && currentRun.startTime== -1L))
    }


//    runs = runSeq.toSet


    val completedRuns = runSeq.filter(_.completed)

    result.duration = completedRuns.map(_.duration).sum
    result.experimentsRun = runSeq.size
    result.ontologiesProcessed = SortedSet[String]()++runSeq.map(_.ontology)
    result.timeOuts = runSeq.size-completedRuns.size
    result.definersLeft = completedRuns.filter(_.definersLeft>0).size
    result.durations = completedRuns.toSeq.map(_.duration)
    result.sizes = completedRuns.map(run => (run.inputSize, run.outputSize))
    result.moduleSizeOutputSize = completedRuns.toSeq.map(run => (run.moduleSize, run.outputSize))
    result.symbolsToForgetDuration = completedRuns.toSeq.map(run => (run.symbolsToForget, run.duration.toInt))
    result.numberRestrictions = completedRuns.toSeq.map(_.numberRestrictions)
    result.runs = runSeq

    result
  }


  def generatePlot(valuePairs: Seq[(Int, Int)], 
		   x: String, 
		   y: String,
		   outputFilename: String,
		   orientationLine: Boolean = true,
		   logScaleX: Boolean = true,
		   logScaleY: Boolean = true
		 ): File = { 
    val dataFile = new File(outputFilename+".txt")//File.createTempFile("data",".txt")

    val graphFile = new File(outputFilename)

    val out = new PrintStream(new FileOutputStream(dataFile, true)) // append

    valuePairs.foreach{ pair => out.println(pair._1.toString+" "+pair._2.toString) }
    
    var argument = ""

    if(logScaleX)
      argument += "set logscale x; "

    if(logScaleY)
      argument += "set logscale y; "

//    argument += "set size ratio 1;"

    argument +=	//"set term pdf; " +
	"set term postscript enhanced color fontscale 2.0; " +
                "set output '"+graphFile.getAbsolutePath+"';" +
                "set xlabel '"+x+"'; set ylabel '"+y+"';" +
		"plot [1:] [1:] '"+dataFile+"' notitle"

    if(orientationLine) 
      argument += ",x notitle ls 3"


    val command = Seq("gnuplot","-e", argument)

//    val res = command !

    println("\\includegraphics[scale=.25]{./"+graphFile.getName+"}")

    graphFile
  }

  def generateDurationsGraph(durations: Seq[Double], outputFilename: String) = { 
    var map: Map[Double, Long] = Map()

    val sorted = durations.sortWith((a, b) => a<b)
    var count = 0
    sorted.foreach{ duration => count+=1; map=map.updated(duration, count) }

    var map2 = new HashMap[Double, Long]()

    map2.put(0,0)
    Set(0,1,2,3).foreach(num => map2.put(Math.pow(10,(num.toDouble)),0))

    val sortedKeys = map2.keySet.toSeq.sortWith((a,b) => a<b)

//    Set(1,10,100,1000).foreach(map2.put(_,0))



    durations.foreach{duration =>
      // (0 to sortedKeys.size-2).foreach{ idx =>
      // 	val min = sortedKeys(idx)
      // 	val max = sortedKeys(idx+1)	     
      // 	if(duration>min && duration<=max)
      // 	  map2.put(min, map2(min)+1)
      // }
      if(duration<=1.0) map2.put(1, map2(1)+1)
      if(duration>1.0 && duration<=10.0) map2.put(10, map2(10)+1)
      if(duration>10.0 && duration<=100.0) map2.put(100, map2(100)+1)
      if(duration>100.0 && duration<=1000.0) map2.put(1000, map2(1000)+1)
    }

    val dataFile = new File("durations.dat")//File.createTempFile("data",".txt")
    val graphFile = new File(outputFilename) //File.createTempFile("plot",".png")

    val out = new PrintStream(new FileOutputStream(dataFile, true)) // append

//    map.toSeq.sorted.foreach{ pair => out.println(pair._1.toString+" "+pair._2.toString) }
    out.println("duration number")
    map2.toSeq.sorted.foreach{ pair => out.println(pair._1.toString+" "+pair._2.toString) }
    
    val command = Seq("gnuplot","-e", "set logscale x; "+
				      "set term postscript enhanced color fontscale 2.0; " + 
		                      "set output '"+graphFile.getAbsolutePath+"';" +
		                      "set xlabel 'Time (s)';"+
				      "set ylabel 'Number of Experiments';" + 
				      "plot '"+dataFile+"' notitle with lines, 0 notitle")



//    val res = command.!

    graphFile
    
  }

  def generateDurationsGraph2(durations: Seq[Double], 
			      outputFilename: String,
			      xLabel: String = "Experiment",
			      yLabel: String = "Time (s)") = { 
    var map: Map[Long, Double] = Map()

    val sorted = durations.sortWith((a, b) => a<b)
    var count = 0L
    sorted.foreach{ value => map=map.updated(count, value); count += 1 }


    val dataFile = new File("durations.txt")
    val graphFile = new File(outputFilename)

    val out = new PrintStream(new FileOutputStream(dataFile, true)) //append

    map.toSeq.sorted.foreach{ pair => out.println(pair._1.toString+" "+pair._2.toString) }
    
    val command = Seq("gnuplot","-e", //"set logscale y; "+
				      "set term postscript enhanced color fontscale 2.0; " +
// 				      "set term pdf; " + 
		                      "set output '"+graphFile.getAbsolutePath+"';" +
		                      "set xlabel '"+xLabel+"';"+
				      "set ylabel '"+yLabel+"';" + 
				      "plot '"+dataFile.getAbsolutePath+"' notitle with lines, " +
				      "0 notitle")


//    val res = command.!

    println("\\includegraphics[scale=.25]{./"+graphFile.getName+"}")

    graphFile
    
  }

}
