aboutsummaryrefslogtreecommitdiff
path: root/huge_page_setup_helper.py
blob: 8bfef141a72d98a3247d95d0f3df32db13315c18 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
#!/usr/bin/python

#
# Tool to set up Linux large page support with minimal effort
#
# by Jarod Wilson <jarod@redhat.com>
# (c) Red Hat, Inc., 2009
#
# Requires hugeadm from libhugetlbfs 2.7 (or backported support)
#
import os

debug = False

# must be executed under the root to operate
if os.geteuid() != 0:
    print "You must be root to setup hugepages!"
    os._exit(1)

# config files we need access to
sysctlConf = "/etc/sysctl.conf"
if not os.access(sysctlConf, os.W_OK):
    print "Cannot access %s" % sysctlConf
    if debug == False:
        os._exit(1)

# This file will be created if it doesn't exist
limitsConf = "/etc/security/limits.d/hugepages.conf"


# Figure out what we've got in the way of memory
memTotal = 0
hugePageSize = 0
hugePages = 0

hugeadmexplain = os.popen("/usr/bin/hugeadm --explain 2>/dev/null").readlines()

for line in hugeadmexplain:
    if line.startswith("Total System Memory:"):
        memTotal = int(line.split()[3])
        break

if memTotal == 0:
    print "Your version of libhugetlbfs' hugeadm utility is too old!"
    os._exit(1)


# Pick the default huge page size and see how many pages are allocated
poolList = os.popen("/usr/bin/hugeadm --pool-list").readlines()
for line in poolList:
    if line.split()[4] == '*':
        hugePageSize = int(line.split()[0])
        hugePages = int(line.split()[2])
        break

if hugePageSize == 0:
    print "Aborting, cannot determine system huge page size!"
    os._exit(1)

# Get initial sysctl settings
shmmax = 0
hugeGID = 0

for line in hugeadmexplain:
    if line.startswith("A /proc/sys/kernel/shmmax value of"):
        shmmax = int(line.split()[4])
        break

for line in hugeadmexplain:
    if line.strip().startswith("vm.hugetlb_shm_group = "):
        hugeGID = int(line.split()[2])
        break


# translate group into textual version
hugeGIDName = "null"
groupNames = os.popen("/usr/bin/getent group").readlines()
for line in groupNames:
    curGID = int(line.split(":")[2])
    if curGID == hugeGID:
        hugeGIDName = line.split(":")[0]
        break


# dump system config as we see it before we start tweaking it
print "Current configuration:"
print " * Total System Memory......: %6d MB" % memTotal
print " * Shared Mem Max Mapping...: %6d MB" % (shmmax / (1024 * 1024))
print " * System Huge Page Size....: %6d MB" % (hugePageSize / (1024 * 1024))
print " * Number of Huge Pages.....: %6d"    % hugePages
print " * Total size of Huge Pages.: %6d MB" % (hugePages * hugePageSize / (1024 * 1024))
print " * Remaining System Memory..: %6d MB" % (memTotal - (hugePages * hugePageSize / (1024 * 1024)))
print " * Huge Page User Group.....:  %s (%d)" % (hugeGIDName, hugeGID)
print


# ask how memory they want to allocate for huge pages
userIn = None
while not userIn:
    try:
        userIn = raw_input("How much memory would you like to allocate for huge pages? "
                           "(input in MB, unless postfixed with GB): ")
        if userIn[-2:] == "GB":
            userHugePageReqMB = int(userIn[0:-2]) * 1024
        elif userIn[-1:] == "G":
            userHugePageReqMB = int(userIn[0:-1]) * 1024
        elif userIn[-2:] == "MB":
            userHugePageReqMB = int(userIn[0:-2])
        elif userIn[-1:] == "M":
            userHugePageReqMB = int(userIn[0:-1])
        else:
            userHugePageReqMB = int(userIn)
        # As a sanity safeguard, require at least 128M not be allocated to huge pages
        if userHugePageReqMB > (memTotal - 128):
            userIn = None
            print "Refusing to allocate %d, you must leave at least 128MB for the system" % userHugePageReqMB
        elif userHugePageReqMB < (hugePageSize / (1024 * 1024)):
            userIn = None
            print "Sorry, allocation must be at least a page's worth!"
        else:
            break
    except ValueError:
        userIn = None
        print "Input must be an integer, please try again!"
userHugePageReqKB = userHugePageReqMB * 1024
userHugePagesReq = userHugePageReqKB / (hugePageSize / 1024)
print "Okay, we'll try to allocate %d MB for huge pages..." % userHugePageReqMB
print


# some basic user input validation
badchars = list(' \\\'":;~`!$^&*(){}[]?/><,')
inputIsValid = False
# ask for the name of the group allowed access to huge pages
while inputIsValid == False:
    foundbad = False
    userGroupReq = raw_input("What group should have access to the huge pages?"
                             "(The group will be created, if need be) [hugepages]: ")
    if userGroupReq is '':
        userGroupReq = 'hugepages'
    if userGroupReq[0].isdigit() or userGroupReq[0] == "-":
        foundbad = True
        print "Group names cannot start with a number or dash, please try again!"
    for char in badchars:
        if char in userGroupReq:
            foundbad = True
            print "Illegal characters in group name, please try again!"
            break
    if len(userGroupReq) > 16:
        foundbad = True
        print "Group names can't be more than 16 characaters, please try again!"
    if foundbad == False:
        inputIsValid = True
print "Okay, we'll give group %s access to the huge pages" % userGroupReq


# see if group already exists, use it if it does, if not, create it
userGIDReq = -1
for line in groupNames:
    curGroupName = line.split(":")[0]
    if curGroupName == userGroupReq:
        userGIDReq = int(line.split(":")[2])
        break

if userGIDReq > -1:
    print "Group %s (gid %d) already exists, we'll use it" % (userGroupReq, userGIDReq)
else:
    if debug == False:
        os.popen("/usr/sbin/groupadd %s" % userGroupReq)
    else:
        print "/usr/sbin/groupadd %s" % userGroupReq
    groupNames = os.popen("/usr/bin/getent group %s" % userGroupReq).readlines()
    for line in groupNames:
        curGroupName = line.split(":")[0]
        if curGroupName == userGroupReq:
            userGIDReq = int(line.split(":")[2])
            break
    print "Created group %s (gid %d) for huge page use" % (userGroupReq, userGIDReq)
print


# basic user input validation, take 2
# space is valid in this case, wasn't in the prior incarnation
badchars = list('\\\'":;~`!$^&*(){}[]?/><,')
inputIsValid = False
# ask for user(s) that should be in the huge page access group
while inputIsValid == False:
    foundbad = False
    userUsersReq = raw_input("What user(s) should have access to the huge pages (space-delimited list, users created as needed)? ")
    for char in badchars:
        if char in userUsersReq:
            foundbad = True
            print "Illegal characters in user name(s) or invalid list format, please try again!"
            break
    for n in userUsersReq.split():
        if len(n) > 32:
            foundbad = True
            print "User names can't be more than 32 characaters, please try again!"
            break
        if n[0] == "-":
            foundbad = True
            print "User names cannot start with a dash, please try again!"
            break
    if foundbad == False:
        inputIsValid = True
# see if user(s) already exist(s)
curUserList = os.popen("/usr/bin/getent passwd").readlines()
hugePageUserList = userUsersReq.split()
for hugeUser in hugePageUserList:
    userExists = False
    for line in curUserList:
        curUser = line.split(":")[0]
        if curUser == hugeUser:
            print "Adding user %s to huge page group" % hugeUser
            userExists = True
            if debug == False:
                os.popen("/usr/sbin/usermod -a -G %s %s" % (userGroupReq, hugeUser))
            else:
                print "/usr/sbin/usermod -a -G %s %s" % (userGroupReq, hugeUser)
        if userExists == True:
            break
    if userExists == False:
        print "Creating user %s with membership in huge page group" % hugeUser
        if debug == False:
            if hugeUser == userGroupReq:
                os.popen("/usr/sbin/useradd %s -g %s" % (hugeUser, userGroupReq))
            else:
                os.popen("/usr/sbin/useradd %s -G %s" % (hugeUser, userGroupReq))
        else:
            print "/usr/sbin/useradd %s -G %s" % (hugeUser, userGroupReq)
print


# set values for the current running environment
if debug == False:
    os.popen("/usr/bin/hugeadm --pool-pages-min DEFAULT:%sM" % userHugePageReqMB)
    os.popen("/usr/bin/hugeadm --pool-pages-max DEFAULT:%sM" % userHugePageReqMB)
    os.popen("/usr/bin/hugeadm --set-shm-group %d" % userGIDReq)
    os.popen("/usr/bin/hugeadm --set-recommended-shmmax")
else:
    print "/usr/bin/hugeadm --pool-pages-min DEFAULT:%sM" % userHugePageReqMB
    print "/usr/bin/hugeadm --pool-pages-max DEFAULT:%sM" % userHugePageReqMB
    print "/usr/bin/hugeadm --set-shm-group %d" % userGIDReq
    print "/usr/bin/hugeadm --set-recommended-shmmax"
    print

# figure out what that shmmax value we just set was
hugeadmexplain = os.popen("/usr/bin/hugeadm --explain 2>/dev/null").readlines()
for line in hugeadmexplain:
    if line.strip().startswith("kernel.shmmax = "):
        shmmax = int(line.split()[2])
        break

# write out sysctl config changes to persist across reboot
if debug == False:
    sysctlConfLines = "# sysctl configuration\n"
    if os.access(sysctlConf, os.W_OK):
        try:
            sysctlConfLines = open(sysctlConf).readlines()
            os.rename(sysctlConf, sysctlConf + ".backup")
            print("Saved original %s as %s.backup" % (sysctlConf, sysctlConf))
        except:
            pass

    fd = open(sysctlConf, "w")
    for line in sysctlConfLines:
        if line.startswith("kernel.shmmax"):
            continue
        elif line.startswith("vm.nr_hugepages"):
            continue
        elif line.startswith("vm.hugetlb_shm_group"):
            continue
        else:
            fd.write(line);

    fd.write("kernel.shmmax = %d\n" % shmmax)
    fd.write("vm.nr_hugepages = %d\n" % userHugePagesReq)
    fd.write("vm.hugetlb_shm_group = %d\n" % userGIDReq)
    fd.close()

else:
    print "Add to %s:" % sysctlConf
    print "kernel.shmmax = %d" % shmmax
    print "vm.nr_hugepages = %d" % userHugePagesReq
    print "vm.hugetlb_shm_group = %d" % userGIDReq
    print


# write out limits.conf changes to persist across reboot
if debug == False:
    limitsConfLines = "# Huge page access configuration\n"
    if os.access(limitsConf, os.W_OK):
        try:
            limitsConfLines = open(limitsConf).readlines()
            os.rename(limitsConf, limitsConf + ".backup")
            print("Saved original %s as %s.backup" % (limitsConf, limitsConf))
        except:
            pass

    fd = open(limitsConf, "w")
    for line in limitsConfLines:
        cfgExist = False
        for hugeUser in hugePageUserList:
            try:
                if line.split()[0] == hugeUser:
                    cfgExist = True
            except IndexError:
                # hit either white or comment line, it is safe not to take
                # any action and continue.
                pass
        if cfgExist == True:
            continue
        else:
            fd.write(line)

    for hugeUser in hugePageUserList:
        fd.write("%s		soft	memlock		%d\n" % (hugeUser, userHugePageReqKB))
        fd.write("%s		hard	memlock		%d\n" % (hugeUser, userHugePageReqKB))
    fd.close()

else:
    print "Add to %s:" % limitsConf
    for hugeUser in hugePageUserList:
        print "%s		soft	memlock		%d" % (hugeUser, userHugePageReqKB)
        print "%s		hard	memlock		%d" % (hugeUser, userHugePageReqKB)


# dump the final configuration of things now that we're done tweaking
print
print "Final configuration:"
print " * Total System Memory......: %6d MB" % memTotal
if debug == False:
    print " * Shared Mem Max Mapping...: %6d MB" % (shmmax / (1024 * 1024))
else:
    # This should be what we *would* have set it to, had we actually run hugeadm --set-recommended-shmmax
    print " * Shared Mem Max Mapping...: %6d MB" % (userHugePagesReq * hugePageSize / (1024 * 1024))
print " * System Huge Page Size....: %6d MB" % (hugePageSize / (1024 * 1024))
print " * Available Huge Pages.....: %6d"    % userHugePagesReq
print " * Total size of Huge Pages.: %6d MB" % (userHugePagesReq * hugePageSize / (1024 * 1024))
print " * Remaining System Memory..: %6d MB" % (memTotal - userHugePageReqMB)
print " * Huge Page User Group.....:  %s (%d)" % (userGroupReq, userGIDReq)
print