-
Notifications
You must be signed in to change notification settings - Fork 0
/
eval_nas.py
69 lines (60 loc) · 2.32 KB
/
eval_nas.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
import random
from nas_prcss import SamplingCellPths,CellPths2Cells,RankingCellPths,CellPthsPredicting,RankingCellPthsByBRP,SiameseRanking
def SpearmanRank(cell_pths):
pred_rank_cells=CellPths2Cells(cell_pths)
pred_ranks=[]
for i,cell in enumerate(pred_rank_cells):
id=cell["id"]
pred_ranks.append([id,i+1])
cell_pths=RankingCellPths(cell_pths,"gt")
gt_rank_cells=CellPths2Cells(cell_pths)
gt_ranks=[]
for i,cell in enumerate(gt_rank_cells):
id=cell["id"]
gt_ranks.append([id,i+1])
pred_ranks=sorted(pred_ranks,key=lambda x:x[0])
gt_ranks=sorted(gt_ranks,key=lambda x:x[0])
ids_len=len(pred_ranks)
ids_diff=0
for i,elemt in enumerate(pred_ranks):
_,pred_rank=elemt
_,gt_rank=gt_ranks[i]
id_diff=(pred_rank-gt_rank)**2
ids_diff+=id_diff
psp=1-((6*ids_diff)/(ids_len*(ids_len**2-1)))
return psp
def MaxAccs(cell_pths):
pred_rank_cells=CellPths2Cells(cell_pths)
max_accs=[]
cur_max_acc=0
for cell in pred_rank_cells:
gt_acc=cell["gt_accuracy"]
if(gt_acc>cur_max_acc):
cur_max_acc=gt_acc
max_accs.append(cur_max_acc)
return max_accs
def EvalPredictor(predictor,cells_dir,all_ops,max_nodes):
all_cell_pths=SamplingCellPths(cells_dir)
CellPthsPredicting(all_cell_pths,predictor,all_ops,max_nodes,False)
all_cell_pths=RankingCellPths(all_cell_pths,"pred")
max_accs=MaxAccs(all_cell_pths)
psp=SpearmanRank(all_cell_pths)
return max_accs,psp
def EvalRandom(cells_dir):
all_cell_pths=SamplingCellPths(cells_dir)
random.shuffle(all_cell_pths)
max_accs=MaxAccs(all_cell_pths)
psp=SpearmanRank(all_cell_pths)
return max_accs,psp
def EvalSiamesePredictor(noec_predictor,ec_predictor,cells_dir,all_ops,max_nodes,k=10):
all_cell_pths=SamplingCellPths(cells_dir)
all_cell_pths=SiameseRanking(all_cell_pths,noec_predictor,ec_predictor,all_ops,max_nodes,k=k)
psp=SpearmanRank(all_cell_pths)
max_accs=MaxAccs(all_cell_pths)
return max_accs,psp
def EvalBRPredictor(br_predictor,cells_dir,all_ops,max_nodes):
all_cell_pths=SamplingCellPths(cells_dir,shuffle=False)
all_cell_pths=RankingCellPthsByBRP(all_cell_pths,br_predictor,all_ops,max_nodes)
max_accs=MaxAccs(all_cell_pths)
psp=SpearmanRank(all_cell_pths)
return max_accs,psp