View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      https://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  
18  /*
19   * This is not the original file distributed by the Apache Software Foundation
20   * It has been modified by the Hipparchus project
21   */
22  package org.hipparchus.stat.correlation;
23  
24  import org.hipparchus.UnitTestUtils;
25  import org.hipparchus.linear.Array2DRowRealMatrix;
26  import org.hipparchus.linear.RealMatrix;
27  import org.hipparchus.random.ISAACRandom;
28  import org.junit.jupiter.api.Test;
29  
30  import static org.junit.jupiter.api.Assertions.assertEquals;
31  
32  class StorelessCovarianceTest {
33  
34      protected final double[] longleyData = new double[] {
35              60323,83.0,234289,2356,1590,107608,1947,
36              61122,88.5,259426,2325,1456,108632,1948,
37              60171,88.2,258054,3682,1616,109773,1949,
38              61187,89.5,284599,3351,1650,110929,1950,
39              63221,96.2,328975,2099,3099,112075,1951,
40              63639,98.1,346999,1932,3594,113270,1952,
41              64989,99.0,365385,1870,3547,115094,1953,
42              63761,100.0,363112,3578,3350,116219,1954,
43              66019,101.2,397469,2904,3048,117388,1955,
44              67857,104.6,419180,2822,2857,118734,1956,
45              68169,108.4,442769,2936,2798,120445,1957,
46              66513,110.8,444546,4681,2637,121950,1958,
47              68655,112.6,482704,3813,2552,123366,1959,
48              69564,114.2,502601,3931,2514,125368,1960,
49              69331,115.7,518173,4806,2572,127852,1961,
50              70551,116.9,554894,4007,2827,130081,1962
51          };
52  
53      protected final double[] swissData = new double[] {
54              80.2,17.0,15,12,9.96,
55              83.1,45.1,6,9,84.84,
56              92.5,39.7,5,5,93.40,
57              85.8,36.5,12,7,33.77,
58              76.9,43.5,17,15,5.16,
59              76.1,35.3,9,7,90.57,
60              83.8,70.2,16,7,92.85,
61              92.4,67.8,14,8,97.16,
62              82.4,53.3,12,7,97.67,
63              82.9,45.2,16,13,91.38,
64              87.1,64.5,14,6,98.61,
65              64.1,62.0,21,12,8.52,
66              66.9,67.5,14,7,2.27,
67              68.9,60.7,19,12,4.43,
68              61.7,69.3,22,5,2.82,
69              68.3,72.6,18,2,24.20,
70              71.7,34.0,17,8,3.30,
71              55.7,19.4,26,28,12.11,
72              54.3,15.2,31,20,2.15,
73              65.1,73.0,19,9,2.84,
74              65.5,59.8,22,10,5.23,
75              65.0,55.1,14,3,4.52,
76              56.6,50.9,22,12,15.14,
77              57.4,54.1,20,6,4.20,
78              72.5,71.2,12,1,2.40,
79              74.2,58.1,14,8,5.23,
80              72.0,63.5,6,3,2.56,
81              60.5,60.8,16,10,7.72,
82              58.3,26.8,25,19,18.46,
83              65.4,49.5,15,8,6.10,
84              75.5,85.9,3,2,99.71,
85              69.3,84.9,7,6,99.68,
86              77.3,89.7,5,2,100.00,
87              70.5,78.2,12,6,98.96,
88              79.4,64.9,7,3,98.22,
89              65.0,75.9,9,9,99.06,
90              92.2,84.6,3,3,99.46,
91              79.3,63.1,13,13,96.83,
92              70.4,38.4,26,12,5.62,
93              65.7,7.7,29,11,13.79,
94              72.7,16.7,22,13,11.22,
95              64.4,17.6,35,32,16.92,
96              77.6,37.6,15,7,4.97,
97              67.6,18.7,25,7,8.65,
98              35.0,1.2,37,53,42.34,
99              44.7,46.6,16,29,50.43,
100             42.8,27.7,22,29,58.33
101         };
102 
103     protected final double[][] longleyDataSimple = {
104         {60323, 83.0},
105         {61122,88.5},
106         {60171, 88.2},
107         {61187, 89.5},
108         {63221, 96.2},
109         {63639, 98.1},
110         {64989, 99.0},
111         {63761, 100.0},
112         {66019, 101.2},
113         {67857, 104.6},
114         {68169, 108.4},
115         {66513, 110.8},
116         {68655, 112.6},
117         {69564, 114.2},
118         {69331, 115.7},
119         {70551, 116.9}
120     };
121 
122     @Test
123     void testLonglySimpleVar(){
124         double rCov = 12333921.73333333246;
125         StorelessBivariateCovariance cov = new StorelessBivariateCovariance();
126         for(int i=0;i<longleyDataSimple.length;i++){
127             cov.increment(longleyDataSimple[i][0],longleyDataSimple[i][0]);
128         }
129         UnitTestUtils.customAssertEquals("simple covariance test", rCov, cov.getResult(), 10E-7);
130     }
131 
132     @Test
133     void testLonglySimpleCov(){
134         double rCov = 36796.660000;
135         StorelessBivariateCovariance cov = new StorelessBivariateCovariance();
136         for(int i=0;i<longleyDataSimple.length;i++){
137             cov.increment(longleyDataSimple[i][0], longleyDataSimple[i][1]);
138         }
139         UnitTestUtils.customAssertEquals("simple covariance test", rCov, cov.getResult(), 10E-7);
140     }
141 
142     /**
143      * Test Longley dataset against R.
144      * Data Source: J. Longley (1967) "An Appraisal of Least Squares
145      * Programs for the Electronic Computer from the Point of View of the User"
146      * Journal of the American Statistical Association, vol. 62. September,
147      * pp. 819-841.
148      *
149      * Data are from NIST:
150      * <a href="https://www.itl.nist.gov/div898/strd/lls/data/LINKS/DATA/Longley.dat">Longley dataset</a>
151      */
152     @Test
153     void testLonglyByRow() {
154         RealMatrix matrix = createRealMatrix(longleyData, 16, 7);
155 
156         double[] rData = new double[] {
157          12333921.73333333246, 3.679666000000000e+04, 343330206.333333313,
158          1649102.666666666744, 1117681.066666666651, 23461965.733333334, 16240.93333333333248,
159          36796.66000000000, 1.164576250000000e+02, 1063604.115416667,
160          6258.666250000000, 3490.253750000000, 73503.000000000, 50.92333333333334,
161          343330206.33333331347, 1.063604115416667e+06, 9879353659.329166412,
162          56124369.854166664183, 30880428.345833335072, 685240944.600000024, 470977.90000000002328,
163          1649102.66666666674, 6.258666250000000e+03, 56124369.854166664,
164          873223.429166666698, -115378.762499999997, 4462741.533333333, 2973.03333333333330,
165          1117681.06666666665, 3.490253750000000e+03, 30880428.345833335,
166          -115378.762499999997, 484304.095833333326, 1764098.133333333, 1382.43333333333339,
167          23461965.73333333433, 7.350300000000000e+04, 685240944.600000024,
168          4462741.533333333209, 1764098.133333333302, 48387348.933333330, 32917.40000000000146,
169          16240.93333333333, 5.092333333333334e+01, 470977.900000000,
170          2973.033333333333, 1382.433333333333, 32917.40000000, 22.66666666666667
171         };
172 
173         StorelessCovariance covMatrix = new StorelessCovariance(7);
174         for(int i=0;i<matrix.getRowDimension();i++){
175             covMatrix.increment(matrix.getRow(i));
176         }
177 
178         RealMatrix covarianceMatrix = covMatrix.getCovarianceMatrix();
179 
180         UnitTestUtils.customAssertEquals("covariance matrix", createRealMatrix(rData, 7, 7), covarianceMatrix, 10E-7);
181 
182     }
183 
184     /**
185      * Test R Swiss fertility dataset against R.
186      * Data Source: R datasets package
187      */
188     @Test
189     void testSwissFertilityByRow() {
190          RealMatrix matrix = createRealMatrix(swissData, 47, 5);
191 
192          double[] rData = new double[] {
193            156.0424976873265, 100.1691489361702, -64.36692876965772, -79.7295097132285, 241.5632030527289,
194            100.169148936170251, 515.7994172062905, -124.39283071230344, -139.6574005550416, 379.9043755781684,
195            -64.3669287696577, -124.3928307123034, 63.64662349676226, 53.5758556891767, -190.5606105457909,
196            -79.7295097132285, -139.6574005550416, 53.57585568917669, 92.4560592044403, -61.6988297872340,
197             241.5632030527289, 379.9043755781684, -190.56061054579092, -61.6988297872340, 1739.2945371877890
198          };
199 
200         StorelessCovariance covMatrix = new StorelessCovariance(5);
201         for(int i=0;i<matrix.getRowDimension();i++){
202             covMatrix.increment(matrix.getRow(i));
203         }
204 
205         RealMatrix covarianceMatrix = covMatrix.getCovarianceMatrix();
206 
207         UnitTestUtils.customAssertEquals("covariance matrix", createRealMatrix(rData, 5, 5), covarianceMatrix, 10E-13);
208     }
209 
210     /**
211      * Test symmetry of the covariance matrix
212      */
213     @Test
214     void testSymmetry() {
215         RealMatrix matrix = createRealMatrix(swissData, 47, 5);
216 
217         final int dimension = 5;
218         StorelessCovariance storelessCov = new StorelessCovariance(dimension);
219         for(int i=0;i<matrix.getRowDimension();i++){
220             storelessCov.increment(matrix.getRow(i));
221         }
222 
223         double[][] covMatrix = storelessCov.getData();
224         for (int i = 0; i < dimension; i++) {
225             for (int j = i; j < dimension; j++) {
226                 assertEquals(covMatrix[i][j], covMatrix[j][i], 10e-9);
227             }
228         }
229     }
230 
231     /**
232      * Test equality of covariance. chk: covariance of two
233      * samples separately and adds them together. cov: computes
234      * covariance of the combined sample showing both are equal.
235      */
236     @Test
237     void testEquivalence() {
238         int num_sets = 2;
239         StorelessBivariateCovariance cov = new StorelessBivariateCovariance();// covariance of the superset
240         StorelessBivariateCovariance chk = new StorelessBivariateCovariance();// check covariance made by appending covariance of subsets
241 
242         ISAACRandom rand = new ISAACRandom(10L);// Seed can be changed
243         for (int s = 0; s < num_sets; s++) {// loop through sets of samlpes
244             StorelessBivariateCovariance covs = new StorelessBivariateCovariance();
245             for (int i = 0; i < 5; i++) { // loop through individual samlpes.
246                 double x = rand.nextDouble();
247                 double y = rand.nextDouble();
248                 covs.increment(x, y);// add sample to the subset
249                 cov.increment(x, y);// add sample to the superset
250             }
251            chk.append(covs);
252         }
253 
254         UnitTestUtils.customAssertEquals("covariance subset test", chk.getResult(), cov.getResult(), 10E-7);
255     }
256 
257     protected RealMatrix createRealMatrix(double[] data, int nRows, int nCols) {
258         double[][] matrixData = new double[nRows][nCols];
259         int ptr = 0;
260         for (int i = 0; i < nRows; i++) {
261             System.arraycopy(data, ptr, matrixData[i], 0, nCols);
262             ptr += nCols;
263         }
264         return new Array2DRowRealMatrix(matrixData);
265     }
266 
267 
268 }
269