blob: 50627fbb0aa54e5c826b7a942327a55ff1c375c6 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
#
# SPDX-License-Identifier: GPL-2.0
# (c) 2020,2022, Mathieu Poirier <mathieu.poirier@linaro.org>
#
1 ## Install GNU's parallel utility:
mpoirier@xps15:~/work$ sudo apt-get install parallel
2 ## Install git:
mpoirier@xps15:~/work$ sudo apt-get git
3 ## Add clone tools and add them to system PATH:
mpoirier@xps15:~/work$ git clone https://git.linaro.org/people/mathieu.poirier/tooling.git
mpoirier@xps15:~/work$ cd tooling
mpoirier@xps15:~/work$ ls
mpoirier@xps15:~/work/kwg-523/test$ ls tooling/
get-mainline-commit get-mainline-feature negative-set patchnet-set patchnet-write positive-set README.txt
mpoirier@xps15:~/work/tooling$ export PATH=$PATH:/home/mpoirier/work/tooling
mpoirier@xps15:~/work/tooling$ chmod 755 *
4 ## Download the PatchNetTool model:
mpoirier@xps15:~/work$ mkdir patchnet
mpoirier@xps15:~/work$ cd patchnet
mpoirier@xps15:~/work/patchnet$ git init .
mpoirier@xps15:~/work/patchnet$ git remote add hvdthong/PatchNetTool.git https://github.com/hvdthong/PatchNetTool.git
mpoirier@xps15:~/work/patchnet$ git fetch hvdthong/PatchNetTool.git
mpoirier@xps15:~/work/patchnet$ git checkout -b PatchNetTool.git/master hvdthong/PatchNetTool.git/master
5 ## Download the PatchNet_updated model:
mpoirier@xps15:~/work$ mkdir patchnet_updated
mpoirier@xps15:~/work$ cd patchnet_updated
mpoirier@xps15:~/work/patchnet_updated$ git init .
mpoirier@xps15:~/work/patchnet_updated$ git remote add Xin-Zhou-smu/PatchNet_updated.git https://github.com/Xin-Zhou-smu/PatchNet_updated.git
mpoirier@xps15:~/work/patchnet_updated$ git fetch Xin-Zhou-smu/PatchNet_updated.git
mpoirier@xps15:~/work/patchnet_updated$ git checkout -b PatchNet_updated.git/patch-1 Xin-Zhou-smu/PatchNet_updated.git/patch-1
6 ## Create kernel directory and download source:
mpoirier@xps15:~/work$ mkdir kernel
mpoirier@xps15:~/work$ cd kernel
mpoirier@xps15:~/work/kernel$ git init .
mpoirier@xps15:~/work/kernel$ git remote add torvalds/linux.git git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
mpoirier@xps15:~/work/kernel$ git remote
torvalds/linux.git
mpoirier@xps15:~/work/kernel$ git fetch torvalds/linux.git
7 ## Checkout the latest version:
mpoirier@xps15:~/work/kernel$ git branch -r
torvalds/linux.git/master
mpoirier@xps15:~/work/kernel$ git checkout -b torvalds/linux.git/master-v5.8 v5.8
Switched to a new branch 'torvalds/linux.git/master-v5.8'
8 ## Get a handle on the stable trees:
IMPORTANT: For the time being the name used for the stable trees must be the
same as what follows. See script "positive-set" for details.
mpoirier@xps15:~/work/kernel$ git remote add stable/linux-stable.git git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
mpoirier@xps15:~/work/kernel$ git fetch stable/linux-stable.git
mpoirier@xps15:~/work/kernel$ git branch -r | grep linux-stable
stable/linux-stable.git/linux-2.6.11.y
stable/linux-stable.git/linux-2.6.12.y
stable/linux-stable.git/linux-2.6.13.y
...
...
...
stable/linux-stable.git/linux-5.5.y
stable/linux-stable.git/linux-5.6.y
stable/linux-stable.git/linux-5.7.y
stable/linux-stable.git/master
9 ## Generate the positive training set by gathering all the patches that have
been backported to stable kernels:
mpoirier@xps15:~/work/kernel$ positive-set positive.txt
(depending on the machine, this can take hours to complete)
mpoirier@xps15:~/work/kernel$ wc -l positive.txt
44397 positive.txt
10 ## Generate the negative training set using the output, i.e file positive.txt,
from the above command:
mpoirier@xps15:~/work/kernel$ negative-set positive.txt negative.txt v5.0..v5.8
(depending on the machine, this can take hours to complete)
mpoirier@xps15:~/work/kernel$ wc -l negative.txt
82121 negative.txt
11 ## Generate the input file to train PatchNetTool by using the output of the above
two commands:
mpoirier@xps15:~/work/kernel$ patchnet-set -p positive.txt -n negative.txt train.txt
mpoirier@xps15:~/work/kernel$ wc -l train.txt
126518 training-set.txt
12 ## Generate the input file to test new patches:
mpoirier@xps15:~/work/kernel$ testing-set testing.txt v5.8..v5.9
13 ## Use the training and test sets throught the preprocessing phase:
mpoirier@xps15:~/work$ cp kernel/train.txt patchnet/preprocessing/
mpoirier@xps15:~/work$ cp kernel/test.txt patchnet/preprocessing/
mpoirier@xps15:~/work$ cd patchnet
mpoirier@xps15:~/work/patchnet/preprocesing$ ./getinfo -j 36 --commit-list train.txt --git /home/linaro/kernel -o train
(Depeding on the machine and the size of the training set, this can take hours)
mpoirier@xps15:~/work/patchnet/preprocessing$ ./getinfo -j 36 --commit-list testing.txt --git /home/linaro/kernel -o testing
(Depeding on the machine and the size of the training set, this can take hours)
14 ## Generate .pkl data from .out files:
mpoirier@xps15:~/work/patchnet/preprocessing$ cp train.out testing.out ../../patchnet_updated/
mpoirier@xps15:~/work/patchnet$ cd ../../patchnet_updated/
mpoirier@xps15:~/work/patchnet_updated$ python text2dict.py -text_path train.out -dict_path train.pkl
mpoirier@xps15:~/work/patchnet_updated$ python text2dict.py -text_path 'testing.out' -dict_path 'test.pkl'
mpoirier@xps15:~/work/patchnet_updated$ python generate_dict.py -text_path1 train.out -text_path2 testing.out -dict_path dict.pkl
15 ## Train the model:
mpoirier@xps15:~/work/patchnet_updated$ python main.py -train -train_data train.pkl -dictionary_data dict.pkl
|
