aboutsummaryrefslogtreecommitdiff
path: root/lava_android_test/test_definitions/blackbox.py
blob: 790569be31cd3df1d2de2030e93034dd4901ee37 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
# Copyright (c) 2012 Linaro Limited

# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
#
# This file is part of LAVA Android Test.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Bridge for the black-box testing implemented by lava-blackbox.
It covers all the available tests in AOSP.
The list is available here from APMTest to ZipFileROTest

**Sample Result URL:** http://validation.linaro.org/lava-server/dashboard/image-reports/linaro-android-member-ti_panda-linaro

**URL:** https://github.com/zyga/lava-blackbox

**Default options:** None
"""

import datetime
import functools
import logging
import os
import pdb
import shutil
import subprocess
import tempfile

from linaro_dashboard_bundle.evolution import DocumentEvolution
from linaro_dashboard_bundle.io import DocumentIO

from lava_android_test.config import get_config


def debuggable_real(func):
    """
    Helper for debugging functions that otherwise have their exceptions
    consumed by the caller. Any exception raised from such a function will
    trigger a pdb session when 'DEBUG_DEBUGGABLE' environment is set.
    """
    @functools.wraps(func)
    def debuggable_decorator(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except:
            logging.exception("exception in @debuggable function")
            pdb.post_mortem()
    return debuggable_decorator


def debuggable_noop(func):
    return func


if os.getenv("DEBUG_DEBUGGABLE"):
    debuggable = debuggable_real
else:
    debuggable = debuggable_noop


class SuperAdb(object):
    """
    Class that implements certain parts of ADB()-like API differently.
    """

    def __init__(self, stock_adb):
        # Name of the adb executable with any required arguments,
        # such as -s 'serial'
        self._adb_cmd = stock_adb.adb.split()

    def __call__(self, command, *args, **kwargs):
        """
        Invoke adb command.

        This call is somewhat special that it wraps two subprocess helper
        functions: check_call and check_output. They are called depending
        on the keyword argument 'stdout', if passed as None then output
        is _not_ saved and is instantly streamed to the stdout of the running
        program. In any other case stdout is buffered and saved, then returned
        """
        cmd = self._adb_cmd + command
        if "stdout" in kwargs and kwargs['stdout'] is None:
            del kwargs['stdout']
            return subprocess.check_call(cmd, *args, **kwargs)
        else:
            return subprocess.check_output(cmd, *args, **kwargs)

    def listdir(self, dirname):
        """
        List directory entries on the android device.

        Similar to adb.listdir() as implemented in ADB() but generates
        subsequent lines instead of returning a big lump of text for the
        developer to parse. Also, instead of using 'ls' on the target it
        uses the special 'ls' command built into adb.

        The two special entries, . and .., are omitted
        """
        for line in self(['ls', dirname]).splitlines():
            # a, b and c are various pieces of stat data
            # but we don't need that here.
            a, b, c, pathname = line.split(' ', 3)
            if pathname not in ('.', '..'):
                yield pathname


class AdbMixIn(object):
    """
    Mix-in class that assists in setting up ADB.

    lava-android-test uses the setadb()/getadb() methods to pass the ADB object
    (which encapsulates connection data for the specific device we will be
    talking to).

    Since the ADB object has fixed API and changes there are beyond the scope
    of this test any extra stuff we want from ADB will be provided by the
    SuperAdb class.

    This mix-in class that has methods expected by lava-android-test and
    exposes two properties, adb and super_adb.
    """

    adb = None

    def setadb(self, adb=None):
        if self.adb is None and adb is not None:
            self.adb = adb
        else:
            self.adb = adb
        self.super_adb = SuperAdb(adb)

    def getadb(self):
        return self.adb


class Sponge(object):
    """
    A simple namespace-like object that anyone can assign and read freely.

    To get some understanding of what is going on both reads and writes are
    logged.
    """

    def __getattr__(self, attr):
        return super(Sponge, self).__getattr__(attr)

    def __setattr__(self, attr, value):
        super(Sponge, self).__setattr__(attr, value)


class FutureFormatDetected(Exception):
    """
    Exception raised when the code detects a new, unsupported
    format that was created after this library was written.

    Since formats do not have partial ordering we can only detect
    a future format when the document format is already at the "latest"
    value, as determined by DocumentEvolution.is_latest(), but the actual
    format is not known to us.

    Typically this won't happen often as document upgrades are not performed
    unless necessary. The only case when this may happen is where the bundle
    loaded from the device was already using a future format to begin with.
    """

    def __init__(self, format):
        self.format = format

    def __str__(self):
        "Unsupported, future format: %s" % self.format

    def __repr__(self):
        return "FutureFormatDetected(%r)" % self.format


class BlackBoxTestBridge(AdbMixIn):
    """
    Bridge for interacting with black box tests implemented as something that
    looks like android test definition.
    """

    # NOTE: none of the tests will actually carry this ID, it is simply used
    # here so that it's not a magic value.
    testname = 'blackbox'

    def __init__(self):
        """
        Initialize black-box test bridge
        """
        # The sponge object is just a requirement from the API, it is not
        # actually used by us in any way. The framework assigns a skeleton
        # test result there but we don't really need it. The Sponge object
        # is a simple 'bag' or namespace that will happily accept and record
        # any values.
        self.parser = Sponge()

    def install(self, install_options=None):
        """
        "Install" blackbox on the test device.

        Black box tests cannot be installed, they must be pre-baked into the
        image. To conform to the 'protocol' used by lava-android-test we will
        perform a fake 'installation' of the black box tests by creating a
        directory that lava-android-test is checking for. We do that only if
        the lava-blackbox executable, which is the entry point to black box
        tests exists in the image.

        ..note::
            This method is part of the lava-android-test framework API.
        """
        if not self.adb.exists(self._blackbox_pathname):
            # Sadly lava-android-test has no exception hierarchy that we can
            # use so all problems are reported as RuntimeError
            raise RuntimeError(
                'blackbox test cannot be "installed" as they must be built'
                ' into the image.'
                ' See https://github.com/zyga/android-lava-wrapper'
                ' for details.')
        else:
            self.adb.makedirs(self._fake_install_path)

    def uninstall(self):
        """
        Conformance method to keep up with the API required by
        lava-android-test. It un-does what install() did by removing the
        _fake_install_path directory from the device.

        ..note::
            This method is part of the lava-android-test framework API.
        """
        if self.adb.exists(self._fake_install_path):
            self.adb.rmtree(self._fake_install_path)

    @debuggable
    def run(self, quiet=False, run_options=None):
        """
        Run the black-box test on the target device.

        Use ADB to run the black-box executable on the device. Keep the results
        in the place that lava-android-test expects us to use.

        ..note::
            This method is part of the lava-android-test framework API.
        """
        # The blackbox test runner will create a directory each time it is
        # started. All of those directories will be created relative to a so
        # called spool directory. Instead of using the default spool directory
        # (which can also change) we will use the directory where
        # lava-android-test keeps all of the results.
        spool_dir = get_config().resultsdir_android
        logging.debug("Using spool directory for black-box testing: %r", spool_dir)
        stuff_before = frozenset(self.super_adb.listdir(spool_dir))
        blackbox_command = [
            'shell', self._blackbox_pathname,
            '--spool', spool_dir,
            '--run-all-tests']
        # Let's run the blackbox executable via ADB
        logging.debug("Starting black-box tests...")
        self.super_adb(blackbox_command, stdout=None)
        logging.debug("Black-box tests have finished!")
        stuff_after = frozenset(self.super_adb.listdir(spool_dir))
        # Check what got added to the spool directory
        new_entries = stuff_after - stuff_before
        if len(new_entries) == 0:
            raise RuntimeError("Nothing got added to the spool directory")
        elif len(new_entries) > 1:
            raise RuntimeError("Multiple items added to the spool directory")
        result_id = list(new_entries)[0]
        print "The blackbox test have finished running, the result id is %r" % result_id
        return result_id

    def parse(self, result_id):
        """
        UNIMPLEMENTED METHOD

        Sadly this method is never called as lava-android-test crashes before
        it gets to realize it is processing blackbox results and load this
        class. This crash _may_ be avoided by hiding the real results of
        blackbox and instead populating the results directory with dummy test
        results that only let LAVA figure out that blackbox is the test to
        load. Then we could monkey patch other parts and it could be
        implemented.

        ONCE THIS IS FIXED THE FOLLOWING DESCRIPTION SHOULD APPLY

        Parse and save results of previous test run.

        The result_id is a name of a directory on the Android device (
        relative to the resultsdir_android configuration option).

        ..note::
            This method is part of the lava-android-test framework API.
        """
        # Sadly since the goal is integration with lava lab I don't have the
        # time to do it. In the lab we use lava-android-test run -o anyway.
        raise NotImplementedError()

    def _get_combined_bundle(self, result_id):
        """
        Compute the combined bundle of a past run and return it
        """
        config = get_config()
        temp_dir = tempfile.mkdtemp()
        remote_bundle_dir = os.path.join(config.resultsdir_android, result_id)
        try:
            self._copy_all_bundles(remote_bundle_dir, temp_dir)
            bundle = self._combine_bundles(temp_dir)
        finally:
            shutil.rmtree(temp_dir)
        return bundle

    # Desired format name, used in a few methods below
    _desired_format = "Dashboard Bundle Format 1.3"

    def _copy_all_bundles(self, android_src, host_dest):
        """
        Use adb pull to copy all the files from android_src (android
        fileystem) to host_dest (host filesystem).
        """
        logging.debug("Saving bundles from %s to %s", android_src, host_dest)
        for name in self.super_adb.listdir(android_src):
            logging.debug("Considering file %s", name)
            # NOTE: We employ simple filtering for '.json' files. This prevents
            # spurious JSON parsing errors if the result directory has
            # additional files of any kind.
            #
            # We _might_ want to lessen that eventually restriction but at this
            # time blackbox is really designed to be self-sufficient so there
            # is no point of additional files.
            if not name.endswith('.json'):
                continue
            remote_pathname = os.path.join(android_src, name)
            local_pathname = os.path.join(host_dest, name)
            try:
                logging.debug(
                    "Copying %s to %s", remote_pathname, local_pathname)
                self.adb.pull(remote_pathname, local_pathname)
            except:
                logging.exception("Unable to copy bundle %s", name)

    def _combine_bundles(self, dirname):
        """
        Combine all bundles from a previous test run into one bundle.

        Returns the aggregated bundle object

        Load, parse and validate each bundle from the specified directory and
        combine them into one larger bundle. This is somewhat tricky. Each
        bundle we coalesce may be generated by a different, separate programs
        and may, thus, use different formats.

        To combine them all correctly we need to take two precautions:
        1) All bundles must be updated to a single, common format
        2) No bundle may be upgraded beyond the latest format known
           to this code. Since the hypothetical 2.0 format may be widely
           different that we cannot reliably interpret anything beyond
           the format field. To prevent this we use the evolution API
           to carefully upgrade only to the "sentinel" format, 1.3
           (at this time)
        """
        # Use DocumentIO.loads() to preserve the order of entries.
        # This is a very small touch but it makes reading the results
        # far more pleasant.
        aggregated_bundle = DocumentIO.loads(
            '{\n'
            '"format": "' + self._desired_format + '",\n'
            '"test_runs": []\n'
            '}\n')[1]
        # Iterate over all files there
        for name in os.listdir(dirname):
            bundle_pathname = os.path.join(dirname, name)
            # Process bundle one by one
            try:
                format, bundle = self._load_bundle(bundle_pathname)
                self._convert_to_common_format(format, bundle)
                self._combine_with_aggregated(aggregated_bundle, bundle)
            except:
                logging.exception("Unable to process bundle %s", name)
        # Return the aggregated bundle
        return aggregated_bundle

    def _load_bundle(self, local_pathname):
        """
        Load the bundle from local_pathname.

        There are various problems that can happen here but
        they should all be treated equally, the bundle not
        being used. This also transparently does schema validation
        so the chance of getting wrong data is lower.
        """
        with open(local_pathname, 'rt') as stream:
            format, bundle = DocumentIO.load(stream)
            return format, bundle

    def _convert_to_common_format(self, format, bundle):
        """
        Convert the bundle to the common format.

        This is a careful and possibly fragile process that may
        raise FutureFormatDetected exception. If that happens
        then desired_format (encoded in the function itself) must be
        changed and the code reviewed for any possible changes
        required to support the more recent format.
        """
        while True:
            # Break conditions, encoded separately for clarity
            if format == self._desired_format:
                # This is our desired break condition, when format
                # becomes (or starts as) the desired format
                break
            if DocumentEvolution.is_latest(bundle):
                # This is a less desired break condition, if we
                # got here then the only possible explanation is
                # that some program started with format > desired_format
                # and the DocumentEvolution API is updated to understand
                # it but we are not. In that case let's raise an exception
                raise FutureFormatDetected(format)
            # As long as the document format is old keep upgrading it
            # step-by-step. Evolution is done in place
            DocumentEvolution.evolve_document(bundle, one_step=True)

    def _combine_with_aggregated(self, aggregated_bundle, bundle):
        """
        Combine the bundle with the contents of aggregated_bundle.

        This method simply transplants all the test runs as that is what
        the bundle format was designed to be - a simple container for test
        runs.
        """
        assert bundle["format"] == self._desired_format
        assert aggregated_bundle["format"] == self._desired_format
        aggregated_bundle["test_runs"].extend(bundle.get("test_runs", []))

    @property
    def _blackbox_pathname(self):
        """
        The path to the blackbox bridge on the device.
        """
        return "/system/bin/lava-blackbox"

    @property
    def _fake_install_path(self):
        """
        The path that we create on the android system to
        indicate that the black box test is installed.

        This is used by uninstall() and install()
        """
        config = get_config()
        return os.path.join(config.installdir_android, self.testname)

    def _monkey_patch_lava(self):
        """
        Monkey patch the implementation of lava_android_test.commands.generate_bundle

        This change is irreversible but given the one-off nature of
        lava-android-test this is okay. It should be safe to do this since
        LAVA will only load the blackbox test module if we explicitly request
        to run it. At that time no other tests will run in the same process.

        This method should not be used once lava-android-test grows a better
        API to allow us to control how bundles are generated.
        """
        from lava_android_test import commands
        def _phony_generate_bundle(serial=None, result_id=None,
                   test=None, test_id=None, attachments=[]):
            if result_id is None:
                raise NotImplementedError
            return self._get_combined_bundle(result_id)
        commands.generate_bundle = _phony_generate_bundle 
        logging.warning(
            "The 'blackbox' test definition has monkey-patched the function"
            " lava_android_test.commands.generate_bundle() if you are _not_"
            " running the blackbox test or are experiencing odd problems/crashes"
            " below please look at this method first")


# initialize the blackbox test definition object
testobj = BlackBoxTestBridge()

# Then monkey patch lava-android-test so that parse keeps working
testobj._monkey_patch_lava()