[PATCH RFC 12/59] Basic 'report' functionality

[Ronald Rojas <ronladred@gmail.com>]

From: George Dunlap <george.dunlap@citrix.com>

Go through the raw data and re-calculate throughput for each
timeframe.  Record for each worker the min and max of these; then take
an average over the whole run.

Have the report go through each and show the worker setup, as well as
the avg, min, and max for each worker.

Also default to running Xen again (rather than processes), and clean
up some chattiness wrt starting Xen processes.

Finally, make the "plan" slightly more programmatic, and test a more
"pipelined" testcase.

Signed-off-by: George Dunlap <george.dunlap@citrix.com>
---
 benchmark.go | 163 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 main.go      |  56 ++++++++++++++++----
 xenworker.go |   4 +-
 3 files changed, 205 insertions(+), 18 deletions(-)

diff --git a/benchmark.go b/benchmark.go
index b2b2399..ec62c3d 100644
--- a/benchmark.go
+++ b/benchmark.go
@@ -9,6 +9,12 @@ import (
 	"encoding/json"
 )
 
+type WorkerSummary struct {
+	MaxTput float64
+	AvgTput float64
+	MinTput float64
+}
+
 type WorkerReport struct {
 	Id int
 	Now int
@@ -43,6 +49,14 @@ type WorkerState struct {
 	LastReport WorkerReport
 }
 
+func Throughput(lt int, lm int, t int, m int) (tput float64) {
+	time := float64(t - lt) / SEC
+	mops := m - lm
+	
+	tput = float64(mops) / time
+	return
+}
+
 func Report(ws *WorkerState, r WorkerReport) {
 	//fmt.Println(r)
 
@@ -52,8 +66,8 @@ func Report(ws *WorkerState, r WorkerReport) {
 		time := float64(r.Now - lr.Now) / SEC
 		mops := r.Mops - lr.Mops
 
-		tput := float64(mops) / time
-
+		tput := Throughput(lr.Now, lr.Mops, r.Now, r.Mops)
+		
 		fmt.Printf("%d Time: %2.3f Mops: %d Tput: %4.2f\n", r.Id, time, mops, tput);
 	}
 
@@ -114,19 +128,21 @@ func NewWorkerList(workers []WorkerSet, workerType int) (ws WorkerList, err erro
 }
 
 type BenchmarkRunData struct {
-	Raw []WorkerReport
+	WorkerCount int
+	Raw []WorkerReport       `json:",omitempty"`
+	Summary []WorkerSummary  `json:",omitempty"`
 }
 
 type BenchmarkRun struct {
-	Completed bool
 	Label string
 	Workers []WorkerSet
 	RuntimeSeconds int
+	Completed bool
 	Results BenchmarkRunData 
 }
 
 func (run *BenchmarkRun) Run() (err error) {
-	Workers, err := NewWorkerList(run.Workers, WorkerProcess)
+	Workers, err := NewWorkerList(run.Workers, WorkerXen)
 	if err != nil {
 		fmt.Println("Error creating workers: %v", err)
 		return
@@ -140,6 +156,8 @@ func (run *BenchmarkRun) Run() (err error) {
 	
 	i := Workers.Start(report, done)
 
+	run.Results.WorkerCount = i
+
 	// FIXME:
 	// 1. Make a zero timeout mean "never"
 	// 2. Make the signals / timeout thing a bit more rational; signal then timeout shouldn't hard kill
@@ -178,8 +196,121 @@ func (run *BenchmarkRun) Run() (err error) {
 	return
 }
 
+func (run *BenchmarkRun) checkSummary() (done bool, err error) {
+	if run.Results.WorkerCount == 0 {
+		err = fmt.Errorf("Internal error: WorkerCount 0!")
+		return
+	}
+	
+	if len(run.Results.Summary) == run.Results.WorkerCount {
+		done = true
+		return 
+	}
+	
+	if len(run.Results.Summary) != 0 {
+		err = fmt.Errorf("Internal error: len(Summary) %d, len(Workers) %d!\n",
+			len(run.Results.Summary), run.Results.WorkerCount)
+		return
+	}
+
+	return
+}
+
+func (run *BenchmarkRun) Process() (err error) {
+	done, err := run.checkSummary()
+	if done || err != nil {
+		return
+	}
+	
+	wcount := run.Results.WorkerCount
+
+	if len(run.Results.Summary) != 0 {
+		err = fmt.Errorf("Internal error: len(Summary) %d, len(Workers) %d!\n",
+			len(run.Results.Summary), wcount)
+		return
+	}
+
+	run.Results.Summary = make([]WorkerSummary, wcount)
+
+	// FIXME: Filter out results which started before all have started
+	
+	data := make([]struct{
+		startTime int
+		lastTime int
+		lastMops int}, wcount)
+
+	for i := range run.Results.Raw {
+		e := run.Results.Raw[i]
+		if e.Id > wcount {
+			err = fmt.Errorf("Internal error: id %d > wcount %d", e.Id, wcount)
+			return
+		}
+		
+		d := &data[e.Id]
+		s := &run.Results.Summary[e.Id]
+
+		if d.startTime == 0 {
+			d.startTime = e.Now
+		} else {
+			tput := Throughput(d.lastTime, d.lastMops, e.Now, e.Mops)
+		
+			if tput > s.MaxTput {
+				s.MaxTput = tput
+			}
+			if tput < s.MinTput || s.MinTput == 0 {
+				s.MinTput = tput
+			}
+		}
+		d.lastTime = e.Now
+		d.lastMops = e.Mops
+	}
+
+	for i := range data {
+		run.Results.Summary[i].AvgTput = Throughput(data[i].startTime, 0, data[i].lastTime, data[i].lastMops)
+	}
+	
+	return
+}
+
+func (run *BenchmarkRun) TextReport() (err error) {
+	var done bool
+	done, err = run.checkSummary()
+	if err != nil {
+		return
+	}
+	if ! done {
+		err = fmt.Errorf("Run not yet processed")
+		return
+	}
+
+	fmt.Printf("== RUN %s ==", run.Label)
+
+	fmt.Printf(" Workers (%d total):\n", run.Results.WorkerCount)
+	wStart := 0
+	for i := range run.Workers {
+		ws := &run.Workers[i]
+		n := ws.Count
+		params := ""
+		for _, s := range ws.Params.Args {
+			params = fmt.Sprintf("%s %s", params, s)
+		}
+		fmt.Printf("[%d-%d]: %s\n", wStart, wStart+n-1, params)
+		wStart += n
+	}
+
+	fmt.Printf("\n%8s %8s %8s %8s\n", "id", "avg", "min", "max")
+	for i := 0; i < run.Results.WorkerCount; i++ {
+		s := &run.Results.Summary[i]
+		fmt.Printf("%8d %8.2f %8.2f %8.2f\n",
+			i, s.AvgTput, s.MinTput, s.MaxTput)
+	}
+
+	return
+}
+
 type BenchmarkPlan struct {
 	filename string
+	WorkerType int
 	Runs []BenchmarkRun
 }
 
@@ -254,3 +385,25 @@ func (plan *BenchmarkPlan) Save() (err error) {
 	}
 	return
 }
+
+func (plan *BenchmarkPlan) TextReport() (err error) {
+	for i := range plan.Runs {
+		r := &plan.Runs[i]
+		if ! r.Completed {
+			fmt.Printf("Test [%d] %s not run\n", i, r.Label)
+		}
+
+		err = r.Process()
+		if err != nil {
+			fmt.Printf("Error processing [%d] %s: %v\n", i, r.Label, err)
+			return
+		}
+
+		err = r.TextReport()
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
diff --git a/main.go b/main.go
index 4d9701c..2696810 100644
--- a/main.go
+++ b/main.go
@@ -10,27 +10,48 @@ func main() {
 
 	switch(os.Args[1]) {
 	case "plan":
+		workerA := []string{"burnwait", "20", "20000000"}
+		//workerB := []string{"burnwait", "10", "20000000"}
+		workerB := []string{"burnwait", "1", "20000000",
+			"burnwait", "2", "20000000",
+			"burnwait", "1", "20000000",
+			"burnwait", "1", "20000000",
+			"burnwait", "1", "20000000",
+			"burnwait", "1", "20000000",
+			"burnwait", "3", "20000000",
+		}
+
+
 		plan :=  BenchmarkPlan{
+			WorkerType:WorkerXen,
 			filename:filename,
 			Runs:[]BenchmarkRun{
 				{Label:"baseline-a",
 					Workers:[]WorkerSet{
-						{Params:WorkerParams{[]string{"burnwait", "20", "20000000"}},
+						{Params:WorkerParams{workerA},
 							Count:1}},
-					RuntimeSeconds:5,},
+					RuntimeSeconds:10,},
 				{Label:"baseline-b",
 					Workers:[]WorkerSet{
-						{Params:WorkerParams{[]string{"burnwait", "10", "20000000"}},
+						{Params:WorkerParams{workerB},
 							Count:1}},
-					RuntimeSeconds:5,},
-				{Label:"4a+4b",
-					Workers:[]WorkerSet{
-						{Params:WorkerParams{[]string{"burnwait", "20", "20000000"}},
-							Count:4},
-						{Params:WorkerParams{[]string{"burnwait", "10", "30000000"}},
-							Count:4}},
-					RuntimeSeconds:5,},
+					RuntimeSeconds:10,},
 			}}
+
+
+		for i := 1; i <= 16 ; i *= 2 {
+			label := fmt.Sprintf("%da+%db", i, i)
+			run := BenchmarkRun{
+				Label:label,
+				Workers:[]WorkerSet{
+					{Params:WorkerParams{workerA},
+						Count:i},
+					{Params:WorkerParams{workerB},
+						Count:i}},
+				RuntimeSeconds:10}
+			plan.Runs = append(plan.Runs, run)
+		}
+		
 		err := plan.Save()
 		if err != nil {
 			fmt.Println("Saving plan ", filename, " ", err)
@@ -49,6 +70,19 @@ func main() {
 			fmt.Println("Running benchmark run:", err)
 			os.Exit(1)
 		}
+		
+	case "report":
+		plan, err := LoadBenchmark(filename)
+		if err != nil {
+			fmt.Println("Loading benchmark ", filename, " ", err)
+			os.Exit(1)
+		}
+	
+		err = plan.TextReport()
+		if err != nil {
+			fmt.Println("Running benchmark run:", err)
+			os.Exit(1)
+		}
 	default:
 		fmt.Println("Unknown argument: ", os.Args[1])
 	}
diff --git a/xenworker.go b/xenworker.go
index 6023c50..1ed5cf0 100644
--- a/xenworker.go
+++ b/xenworker.go
@@ -105,7 +105,7 @@ func (w *XenWorker) Init(p WorkerParams) (err error) {
 			return
 		}
 
-		fmt.Printf(" %s domid %d\n", w.vmname, w.domid)
+		//fmt.Printf(" %s domid %d\n", w.vmname, w.domid)
 	}
 	
 	// Set xenstore config
@@ -133,7 +133,7 @@ func (w *XenWorker) Init(p WorkerParams) (err error) {
 		//fmt.Printf("json:\n%s\n", string(rcfgBytes))
 		rcfgPath := fmt.Sprintf("/local/domain/%d/rumprun/cfg", w.domid)
 
-		fmt.Printf("Writing to %s, json config %s\n", rcfgPath, rcfgBytes)
+		//fmt.Printf("Writing to %s, json config %s\n", rcfgPath, rcfgBytes)
 		
 		args := []string{"xenstore-write", rcfgPath, string(rcfgBytes)}
 		if mock {
-- 
2.7.4


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel