View Javadoc

    1   /**
    2    * <copyright>
    3    *  Copyright 1997-2002 BBNT Solutions, LLC
    4    *  under sponsorship of the Defense Advanced Research Projects Agency (DARPA).
    5    *
    6    *  This program is free software; you can redistribute it and/or modify
    7    *  it under the terms of the Cougaar Open Source License as published by
    8    *  DARPA on the Cougaar Open Source Website (www.cougaar.org).
    9    *
    10   *  THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS
    11   *  PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR
    12   *  IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF
    13   *  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT
    14   *  ANY WARRANTIES AS TO NON-INFRINGEMENT.  IN NO EVENT SHALL COPYRIGHT
    15   *  HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL
    16   *  DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS,
    17   *  TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
    18   *  PERFORMANCE OF THE COUGAAR SOFTWARE.
    19   * </copyright>
    20   *
    21   * Created on Aug 26, 2002
    22   */
    23  package net.sourceforge.pmd.stat;
    24
    25  import static net.sourceforge.pmd.lang.rule.stat.StatisticalRule.MINIMUM_DESCRIPTOR;
    26  import static net.sourceforge.pmd.lang.rule.stat.StatisticalRule.SIGMA_DESCRIPTOR;
    27  import static net.sourceforge.pmd.lang.rule.stat.StatisticalRule.TOP_SCORE_DESCRIPTOR;
    28  import static org.junit.Assert.assertEquals;
    29  import static org.junit.Assert.assertTrue;
    30
    31  import java.util.ArrayList;
    32  import java.util.Collections;
    33  import java.util.Iterator;
    34  import java.util.List;
    35  import java.util.Random;
    36
    37  import junit.framework.AssertionFailedError;
    38  import net.sourceforge.pmd.Report;
    39  import net.sourceforge.pmd.Rule;
    40  import net.sourceforge.pmd.RuleContext;
    41  import net.sourceforge.pmd.lang.Language;
    42  import net.sourceforge.pmd.lang.java.ast.DummyJavaNode;
    43  import net.sourceforge.pmd.lang.java.symboltable.SourceFileScope;
    44  import net.sourceforge.pmd.lang.rule.stat.StatisticalRule;
    45  import net.sourceforge.pmd.stat.DataPoint;
    46  import net.sourceforge.pmd.stat.Metric;
    47
    48  import org.junit.Before;
    49  import org.junit.Ignore;
    50  import org.junit.Test;
    51
    52  /**
    53   * This class tests the Statistical Rules in PMD.
    54   * <p/>
    55   * The idea is, that we fill up 999 datapoints into
    56   * the Stat Rule, and then throw random parameters
    57   * at it.
    58   * <p/>
    59   * The three parameters which are checked are:
    60   * sigma - # Sigmas over the mean.
    61   * topscore - Only the top 5 or so items.
    62   * minimum - Only things of score 10 or better
    63   * <p/>
    64   * When more than one parameter is lumped together, then
    65   * we expect the one which would return the fewest to
    66   * determine what gets sent back.
    67   * <p/>
    68   * So, we throw each collection of parameters, where each
    69   * one is a different order into the system.  We check the
    70   * results off of what the smallest value should be.
    71   * <p/>
    72   * If you are going to work with StatisticalRule any, please
    73   * bump the "NUM_TESTS" number up to something like 128.  That
    74   * way you are more likely to identify problems.  It is set low
    75   * now to make building and running tests easier (when we aren't
    76   * touching the file.)
    77   * <p/>
    78   * Note also, that when verifying the Sigma, I wasn't quite able
    79   * to determine how many results it would return (it would vary
    80   * from -2 to 2 of what I expected.)  That is what the delta
    81   * parameter on the verify method takes.  If you can figure it
    82   * out exactly, (without stealing code from the StatRule) then
    83   * feel free to change it and tighten the deltas.
    84   */
    85  public class StatisticalRuleTest  {
    86
    87      private static final int POINTS = 100;
    88
    89      private DataPoint[] points = new DataPoint[POINTS];
    90      private MockStatisticalRule IUT = null;
    91      private String testName = "";//FIXME - why/when was this added. It was never set.
    92      private Random random = new Random();
    93
    94      public static final double MAX_MINIMUM = POINTS;
    95      public static final double NO_MINIMUM = -1.0;
    96      public static final double MAX_SIGMA = 5.0;
    97      public static final double NO_SIGMA = -1.0;
    98      public static final int MIN_TOPSCORE = 0;
    99      public static final int NO_TOPSCORE = -1;
    100
    101
    102     public static final double MEAN = 49.5;
    103     public static final double SIGMA = 29.0115;
    104     public static final int NUM_TESTS = 1;
    105
    106     public static final double DELTA = 0.005;
    107
    108
    109     @Before
    110     public void setUp() {
    111         IUT = new MockStatisticalRule();
    112         if (testName.endsWith("0")) {
    113             for (int i = 0; i < POINTS; i++) {
    114                 points[i] = new DataPoint();
    115                 points[i].setScore(1.0 * i);
    116                 DummyJavaNode s = new DummyJavaNode(1);
    117                 s.setScope(new SourceFileScope("foo"));
    118                 s.testingOnly__setBeginLine(i);
    119                 s.testingOnly__setBeginColumn(1);
    120                 points[i].setNode(s);
    121                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
    122
    123                 IUT.addDataPoint(points[i]);
    124             }
    125         } else if (testName.endsWith("1")) {
    126             for (int i = POINTS - 1; i >= 0; i--) {
    127                 points[i] = new DataPoint();
    128                 points[i].setScore(1.0 * i);
    129                 DummyJavaNode s = new DummyJavaNode(1);
    130                 s.setScope(new SourceFileScope("foo"));
    131                 s.testingOnly__setBeginLine(i);
    132                 s.testingOnly__setBeginColumn(1);
    133                 points[i].setNode(s);
    134                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
    135
    136                 IUT.addDataPoint(points[i]);
    137             }
    138         } else {
    139             List<DataPoint> lPoints = new ArrayList<DataPoint>();
    140             for (int i = 0; i < POINTS; i++) {
    141                 points[i] = new DataPoint();
    142                 points[i].setScore(1.0 * i);
    143                 DummyJavaNode s = new DummyJavaNode(1);
    144                 s.setScope(new SourceFileScope("foo"));
    145                 s.testingOnly__setBeginLine(i);
    146                 s.testingOnly__setBeginColumn(1);
    147                 s.testingOnly__setBeginColumn(1);
    148                 points[i].setNode(s);
    149                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
    150
    151                 lPoints.add(points[i]);
    152             }
    153
    154             Collections.shuffle(lPoints);
    155             for (int i = 0; i < POINTS; i++) {
    156                 IUT.addDataPoint(lPoints.get(i));
    157             }
    158         }
    159
    160     }
    161
    162     /**
    163      * This test verifies that the Stat rule creates a Metric,
    164      * with the proper values.
    165      */
    166     @Test
    167     public void testMetrics() throws Throwable {
    168         Report report = makeReport(IUT);
    169         Iterator metrics = report.metrics();
    170
    171         assertTrue(metrics.hasNext());
    172         Object o = metrics.next();
    173
    174         assertTrue(o instanceof Metric);
    175         Metric m = (Metric) o;
    176
    177         assertEquals("net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName());
    178
    179         assertEquals(0.0, m.getLowValue(), 0.05);
    180         assertEquals(POINTS - 1.0, m.getHighValue(), 0.05);
    181         assertEquals(MEAN, m.getAverage(), 0.05);
    182         assertEquals(SIGMA, m.getStandardDeviation(), 0.05);
    183     }
    184
    185     /**
    186      * This returns a Random value for Sigma which will
    187      * return some values.
    188      */
    189     public double randomSigma() {
    190         return random.nextDouble() * 1.0;
    191     }
    192
    193     /**
    194      * This returns a Random value for Sigma which value
    195      * is greater than the parameter.
    196      */
    197     public double randomSigma(int minimum) {
    198         double minSigma = ((POINTS - 1 - minimum) - MEAN) / SIGMA;
    199
    200         if ((minSigma <= 0) || (minSigma > 2))
    201             return randomSigma();
    202
    203         return minSigma + (random.nextDouble() * (2 - minSigma));
    204     }
    205
    206     /**
    207      * This returns the expected number of results when
    208      * the Sigma rating is the smallest.
    209      */
    210     public int expectedSigma(double sigma) {
    211         long expectedMin = Math.round(MEAN + (sigma * SIGMA));
    212
    213         if (((POINTS - 1) - expectedMin) < 0)
    214             return 0;
    215         return (POINTS - 1) - (int) expectedMin;
    216     }
    217
    218     /**
    219      * This generates a random minimum value for testing.
    220      */
    221     public double randomMinimum() {
    222         return random.nextDouble() * (POINTS - 1);
    223     }
    224
    225     /**
    226      * This generates a random minimum value for which fewer
    227      * results would be returned.
    228      */
    229     public double randomMinimum(int minimum) {
    230         double diffTarget = 1.0 * (POINTS - 1 - minimum);
    231         return (random.nextDouble() * minimum) + diffTarget;
    232     }
    233
    234     /**
    235      * This returns the expected number of reports.
    236      * <p/>
    237      * If the Minimum comes in at 521.569 then we expect
    238      * 522, 523, ... 999 will pass.
    239      */
    240     public int expectedMinimum(double minimum) {
    241         Double d = Double.valueOf(minimum);
    242         return POINTS - 1 - d.intValue();
    243     }
    244
    245     @Test
    246     public void testExpectedMinimum() {
    247         for (int i = 0; i < POINTS - 1; i++) {
    248             assertEquals("Integer Min", POINTS - 1 - i, expectedMinimum(i * 1.0));
    249             assertEquals("Double Min", POINTS - 1 - i, expectedMinimum((i * 1.0) + 0.5));
    250         }
    251     }
    252
    253     /**
    254      * This returns a random value for Top Score.
    255      */
    256     public int randomTopScore() {
    257         return random.nextInt(POINTS - 1);
    258     }
    259
    260     /**
    261      * This will return a random value for the Top Score
    262      * which will return more than the minimum provided.
    263      */
    264     public int randomTopScore(double target) {
    265         if (target < 0)
    266             return 0;
    267
    268         return random.nextInt(Double.valueOf(target).intValue());
    269     }
    270
    271     /**
    272      * This will return the expected number of results
    273      * with the given Top Score.
    274      */
    275     public int expectedTopScore(int target) {
    276         return target;
    277     }
    278
    279     // Test Single Datapoint
    280     @Test
    281     public void testSingleDatapoint() {
    282         StatisticalRule IUT = new MockStatisticalRule();
    283
    284         DataPoint point = new DataPoint();
    285         point.setScore(POINTS + 1.0);
    286         DummyJavaNode s = new DummyJavaNode(1);
    287         s.setScope(new SourceFileScope("foo"));
    288         s.testingOnly__setBeginLine(POINTS + 1);
    289         s.testingOnly__setBeginColumn(1);
    290         point.setNode(s);
    291         point.setMessage("SingleDataPoint");
    292
    293         IUT.setProperty(MINIMUM_DESCRIPTOR, (double)POINTS);
    294
    295         IUT.addDataPoint(point);
    296
    297         Report report = makeReport(IUT);
    298
    299         assertEquals("Expecting only one result", 1, report.size());
    300     }
    301
    302     // Okay, we have three properties we need to
    303     // test in Combination:
    304     //  S = Sigma
    305     //  T = Top Score
    306     //  M = Minimum
    307     //
    308     // They are listed in decreasing order of what
    309     // to expect.
    310     //
    311     // Thus testSM() should have the Sigma less than
    312     // the minimum, so we expect the Minimum # of results.
    313     //
    314
    315     @Test
    316     public void testS() throws Throwable {
    317         verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2);
    318
    319         for (int i = 0; i < NUM_TESTS; i++) {
    320             double sigma = randomSigma();
    321             verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2);
    322         }
    323     }
    324
    325     @Test
    326     public void testS1() throws Throwable {
    327         testS();
    328     }
    329
    330     @Test
    331     public void testS2() throws Throwable {
    332         testS();
    333     }
    334
    335     @Test
    336     public void testS3() throws Throwable {
    337         testS();
    338     }
    339
    340     @Test
    341     public void testS4() throws Throwable {
    342         testS();
    343     }
    344
    345     @Test
    346     public void testS5() throws Throwable {
    347         testS();
    348     }
    349
    350
    351     @Test
    352     public void testT() throws Throwable {
    353         verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0);
    354
    355         for (int i = 0; i < NUM_TESTS; i++) {
    356             int topScore = randomTopScore();
    357             verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0);
    358         }
    359     }
    360
    361     @Test
    362     public void testT1() throws Throwable {
    363         testT();
    364     }
    365
    366     @Test
    367     public void testT2() throws Throwable {
    368         testT();
    369     }
    370
    371     @Test
    372     public void testT3() throws Throwable {
    373         testT();
    374     }
    375
    376     @Test
    377     public void testT4() throws Throwable {
    378         testT();
    379     }
    380
    381     @Test
    382     public void testT5() throws Throwable {
    383         testT();
    384     }
    385
    386     @Test
    387     public void testM() throws Throwable {
    388         verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0);
    389
    390         for (int i = 0; i < NUM_TESTS; i++) {
    391             double minimum = randomMinimum();
    392             verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0);
    393         }
    394     }
    395
    396     @Test
    397     public void testM1() throws Throwable {
    398         testM();
    399     }
    400
    401     @Test
    402     public void testM2() throws Throwable {
    403         testM();
    404     }
    405
    406     @Test
    407     public void testM3() throws Throwable {
    408         testM();
    409     }
    410
    411     @Test
    412     public void testM4() throws Throwable {
    413         testM();
    414     }
    415
    416     @Test
    417     public void testM5() throws Throwable {
    418         testM();
    419     }
    420
    421     @Test
    422     public void testST() throws Throwable {
    423         verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0);
    424
    425         for (int i = 0; i < NUM_TESTS; i++) {
    426             double sigma = randomSigma();
    427             int topScore = randomTopScore(expectedSigma(sigma));
    428
    429             verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0);
    430         }
    431     }
    432
    433     @Test
    434     public void testST1() throws Throwable {
    435         testST();
    436     }
    437
    438     @Test
    439     public void testST2() throws Throwable {
    440         testST();
    441     }
    442
    443     @Test
    444     public void testST3() throws Throwable {
    445         testST();
    446     }
    447
    448     @Test
    449     public void testST4() throws Throwable {
    450         testST();
    451     }
    452
    453     @Test
    454     public void testST5() throws Throwable {
    455         testST();
    456     }
    457
    458     @Test
    459     public void testTS() throws Throwable {
    460         verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0);
    461
    462         for (int i = 0; i < NUM_TESTS; i++) {
    463             int topScore = randomTopScore();
    464             double sigma = randomSigma(expectedTopScore(topScore));
    465
    466             verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2);
    467         }
    468     }
    469
    470     @Test
    471     public void testTS1() throws Throwable {
    472         testTS();
    473     }
    474
    475     @Test
    476     public void testTS2() throws Throwable {
    477         testTS();
    478     }
    479
    480     @Test
    481     public void testTS3() throws Throwable {
    482         testTS();
    483     }
    484
    485     @Test
    486     public void testTS4() throws Throwable {
    487         testTS();
    488     }
    489
    490     @Test
    491     public void testTS5() throws Throwable {
    492         testTS();
    493     }
    494
    495     @Test
    496     public void testSM() throws Throwable {
    497         verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0);
    498         for (int i = 0; i < NUM_TESTS; i++) {
    499             double sigma = randomSigma();
    500             double minimum = randomMinimum(expectedSigma(sigma));
    501
    502             verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0);
    503         }
    504
    505     }
    506
    507     @Test
    508     public void testSM1() throws Throwable {
    509         testSM();
    510     }
    511
    512     @Test
    513     public void testSM2() throws Throwable {
    514         testSM();
    515     }
    516
    517     @Test
    518     public void testSM3() throws Throwable {
    519         testSM();
    520     }
    521
    522     @Test
    523     public void testSM4() throws Throwable {
    524         testSM();
    525     }
    526
    527     @Test
    528     public void testSM5() throws Throwable {
    529         testSM();
    530     }
    531
    532
    533     @Test
    534     public void testMS() throws Throwable {
    535         verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0);
    536         for (int i = 0; i < NUM_TESTS; i++) {
    537             double minimum = randomMinimum();
    538             double sigma = randomSigma(expectedMinimum(minimum));
    539
    540             verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2);
    541         }
    542     }
    543
    544     @Test
    545     public void testMS1() throws Throwable {
    546         testMS();
    547     }
    548
    549     @Test
    550     public void testMS2() throws Throwable {
    551         testMS();
    552     }
    553
    554     @Test
    555     public void testMS3() throws Throwable {
    556         testMS();
    557     }
    558
    559     @Test
    560     public void testMS4() throws Throwable {
    561         testMS();
    562     }
    563
    564     @Test
    565     public void testMS5() throws Throwable {
    566         testMS();
    567     }
    568
    569
    570     @Test
    571     public void testTM() throws Throwable {
    572         verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0);
    573         for (int i = 0; i < NUM_TESTS; i++) {
    574             int topScore = randomTopScore();
    575             double minimum = randomMinimum(expectedTopScore(topScore));
    576
    577             verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0);
    578         }
    579     }
    580
    581     @Test
    582     public void testTM1() throws Throwable {
    583         testTM();
    584     }
    585
    586     @Test
    587     public void testTM2() throws Throwable {
    588         testTM();
    589     }
    590
    591     @Test
    592     public void testTM3() throws Throwable {
    593         testTM();
    594     }
    595
    596     @Test
    597     public void testTM4() throws Throwable {
    598         testTM();
    599     }
    600
    601     @Test
    602     public void testTM5() throws Throwable {
    603         testTM();
    604     }
    605
    606
    607     @Test
    608     public void testMT() throws Throwable {
    609         verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0);
    610         for (int i = 0; i < NUM_TESTS; i++) {
    611             double minimum = randomMinimum();
    612             int topScore = randomTopScore(expectedMinimum(minimum));
    613
    614             verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0);
    615         }
    616     }
    617
    618     @Test
    619     public void testMT1() throws Throwable {
    620         testMT();
    621     }
    622
    623     @Test
    624     public void testMT2() throws Throwable {
    625         testMT();
    626     }
    627
    628     @Test
    629     public void testMT3() throws Throwable {
    630         testMT();
    631     }
    632
    633     @Test
    634     public void testMT4() throws Throwable {
    635         testMT();
    636     }
    637
    638     @Test
    639     public void testMT5() throws Throwable {
    640         testMT();
    641     }
    642
    643
    644     @Test
    645     public void testSTM() throws Throwable {
    646         double sigma = randomSigma();
    647         verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0);
    648
    649         for (int i = 0; i < NUM_TESTS; i++) {
    650             sigma = randomSigma();
    651             int topScore = randomTopScore(expectedSigma(sigma));
    652             double minimum = randomMinimum(expectedTopScore(topScore));
    653
    654             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
    655         }
    656     }
    657
    658     @Test
    659     public void testSTM1() throws Throwable {
    660         testSTM();
    661     }
    662
    663     @Test
    664     public void testSTM2() throws Throwable {
    665         testSTM();
    666     }
    667
    668     @Test
    669     public void testSTM3() throws Throwable {
    670         testSTM();
    671     }
    672
    673     @Test
    674     public void testSTM4() throws Throwable {
    675         testSTM();
    676     }
    677
    678     @Test
    679     public void testSTM5() throws Throwable {
    680         testSTM();
    681     }
    682
    683     @Test
    684     public void testSMT() throws Throwable {
    685         double sigma = randomSigma();
    686         verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0);
    687
    688         for (int i = 0; i < NUM_TESTS; i++) {
    689             sigma = randomSigma();
    690             double minimum = randomMinimum(expectedSigma(sigma));
    691             int topScore = randomTopScore(expectedMinimum(minimum));
    692
    693             verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0);
    694         }
    695     }
    696
    697     @Test
    698     public void testSMT1() throws Throwable {
    699         testSMT();
    700     }
    701
    702     @Test
    703     public void testSMT2() throws Throwable {
    704         testSMT();
    705     }
    706
    707     @Test
    708     public void testSMT3() throws Throwable {
    709         testSMT();
    710     }
    711
    712     @Test
    713     public void testSMT4() throws Throwable {
    714         testSMT();
    715     }
    716
    717     @Test
    718     public void testSMT5() throws Throwable {
    719         testSMT();
    720     }
    721
    722     @Test
    723     // because of random failures during continuous integration,
    724     // tests are disabled in regress mode until somebody figures out
    725     // what the tests are supposed to measure and why they sometime fail
    726     @Ignore("random failures during continuous integration")
    727     public void testTSM() throws Throwable {
    728         int topScore = randomTopScore();
    729         verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0);
    730
    731         for (int i = 0; i < NUM_TESTS; i++) {
    732             topScore = randomTopScore();
    733             double sigma = randomSigma(expectedTopScore(topScore));
    734             double minimum = randomMinimum(expectedSigma(sigma));
    735
    736             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
    737         }
    738     }
    739
    740     @Test
    741     @Ignore("random failures during continuous integration")
    742     public void testTSM1() throws Throwable {
    743         testTSM();
    744     }
    745
    746     @Test
    747     @Ignore("random failures during continuous integration")
    748     public void testTSM2() throws Throwable {
    749         testTSM();
    750     }
    751
    752     @Test
    753     @Ignore("random failures during continuous integration")
    754     public void testTSM3() throws Throwable {
    755         testTSM();
    756     }
    757
    758     @Test
    759     @Ignore("random failures during continuous integration")
    760     public void testTSM4() throws Throwable {
    761         testTSM();
    762     }
    763
    764     @Test
    765     @Ignore("random failures during continuous integration")
    766     public void testTSM5() throws Throwable {
    767         testTSM();
    768     }
    769
    770     @Test
    771     public void testTMS() throws Throwable {
    772         int topScore = randomTopScore();
    773         verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0);
    774
    775         for (int i = 0; i < NUM_TESTS; i++) {
    776             topScore = randomTopScore();
    777             double minimum = randomMinimum(expectedTopScore(topScore));
    778             double sigma = randomSigma(expectedMinimum(minimum));
    779
    780             verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2);
    781         }
    782     }
    783
    784     @Test
    785     public void testTMS1() throws Throwable {
    786         testTMS();
    787     }
    788
    789     @Test
    790     public void testTMS2() throws Throwable {
    791         testTMS();
    792     }
    793
    794     @Test
    795     public void testTMS3() throws Throwable {
    796         testTMS();
    797     }
    798
    799     @Test
    800     public void testTMS4() throws Throwable {
    801         testTMS();
    802     }
    803
    804     @Test
    805     public void testTMS5() throws Throwable {
    806         testTMS();
    807     }
    808
    809     /**
    810      * Verifies what happens when you pass these parameters
    811      * into the thing.  DELTA is the amount of error allowed.
    812      * Usually DELTA is only used for Sigma, as we really can't
    813      * calculate it exactly.
    814      */
    815
    816     public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) {
    817         try {
    818             setUp();
    819             if (sigma >= 0) {
    820             	IUT.setProperty(SIGMA_DESCRIPTOR, sigma);
    821             }
    822
    823             if (minimum >= 0) {
    824             	IUT.setProperty(MINIMUM_DESCRIPTOR, minimum);
    825             }
    826
    827             if (topScore >= 0) {
    828                 IUT.setProperty(TOP_SCORE_DESCRIPTOR, topScore);
    829             }
    830
    831             Report report = makeReport(IUT);
    832             if (delta == 0) {
    833                 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size());
    834             } else {
    835                 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size();
    836
    837                 assertTrue(assertStr, report.size() >= (expected - delta));
    838                 assertTrue(assertStr, report.size() <= (expected + delta));
    839             }
    840         } catch (AssertionFailedError afe) {
    841             System.err.println("******** " + testName + " ***********");
    842             if (sigma != NO_SIGMA) {
    843                 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma)));
    844             }
    845
    846             if (minimum != NO_MINIMUM) {
    847                 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum)));
    848             }
    849
    850             if (topScore != NO_TOPSCORE) {
    851                 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore)));
    852             }
    853
    854             throw afe;
    855
    856         }
    857     }
    858
    859     public Report makeReport(Rule IUT) {
    860         List list = new ArrayList();
    861         Report report = new Report();
    862
    863         RuleContext ctx = new RuleContext();
    864         ctx.setReport(report);
    865         ctx.setSourceCodeFilename(testName);
    866         ctx.setLanguageVersion(Language.JAVA.getDefaultVersion());
    867
    868         IUT.apply(list, ctx);
    869
    870         return report;
    871     }
    872
    873     public static junit.framework.Test suite() {
    874         return new junit.framework.JUnit4TestAdapter(StatisticalRuleTest.class);
    875     }
    876 }