bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/DebugSupportImpl.java

   1 package fi.vtt.simantics.procore.internal;
   2
   3 import java.io.File;
   4 import java.io.IOException;
   5 import java.lang.management.ManagementFactory;
   6 import java.lang.reflect.Method;
   7 import java.util.HashMap;
   8 import java.util.Map;
   9 import java.util.TreeSet;
  10 import java.util.UUID;
  11
  12 import org.eclipse.core.runtime.Platform;
  13 import org.osgi.framework.Bundle;
  14 import org.simantics.databoard.binding.mutable.Variant;
  15 import org.simantics.databoard.parser.repository.DataValueRepository;
  16 import org.simantics.db.DirectStatements;
  17 import org.simantics.db.Resource;
  18 import org.simantics.db.Session;
  19 import org.simantics.db.Statement;
  20 import org.simantics.db.WriteGraph;
  21 import org.simantics.db.common.request.WriteRequest;
  22 import org.simantics.db.common.request.WriteResultRequest;
  23 import org.simantics.db.common.utils.Logger;
  24 import org.simantics.db.common.utils.NameUtils;
  25 import org.simantics.db.exception.DatabaseException;
  26 import org.simantics.db.exception.ServiceException;
  27 import org.simantics.db.exception.ValidationException;
  28 import org.simantics.db.impl.Activator;
  29 import org.simantics.db.impl.ClusterI;
  30 import org.simantics.db.impl.ResourceImpl;
  31 import org.simantics.db.impl.graph.ReadGraphImpl;
  32 import org.simantics.db.impl.graph.WriteLogger;
  33 import org.simantics.db.impl.query.QueryProcessor;
  34 import org.simantics.db.impl.query.QuerySupport;
  35 import org.simantics.db.procore.cluster.ClusterBig;
  36 import org.simantics.db.procore.cluster.ClusterImpl;
  37 import org.simantics.db.procore.cluster.ClusterSmall;
  38 import org.simantics.db.service.DebugSupport;
  39 import org.simantics.db.service.QueryControl;
  40 import org.simantics.db.service.XSupport;
  41 import org.simantics.layer0.Layer0;
  42 import org.simantics.scl.runtime.function.Function2;
  43 import org.simantics.scl.runtime.function.Function3;
  44 import org.simantics.scl.runtime.function.FunctionImpl2;
  45 import org.simantics.scl.runtime.function.FunctionImpl3;
  46 import org.simantics.utils.Development;
  47 import org.simantics.utils.FileUtils;
  48
  49 import gnu.trove.map.hash.TIntIntHashMap;
  50 import gnu.trove.procedure.TIntIntProcedure;
  51 import gnu.trove.procedure.TIntShortProcedure;
  52 import gnu.trove.set.hash.TIntHashSet;
  53
  54 public class DebugSupportImpl implements DebugSupport {
  55
  56         final private Map<String, Function2<WriteGraph, String, Object>> getCommands = new HashMap<String, Function2<WriteGraph, String, Object>>();
  57         final private Map<String, Function3<WriteGraph, File, String, String>> listCommands = new HashMap<String, Function3<WriteGraph, File, String, String>>();
  58         final private Map<String, Function2<WriteGraph, String, String>> execCommands = new HashMap<String, Function2<WriteGraph, String, String>>();
  59         final private Map<String, Function2<WriteGraph, String, String>> printCommands = new HashMap<String, Function2<WriteGraph, String, String>>();
  60
  61         private static SessionImplSocket getSession(WriteGraph graph) {
  62                 return (SessionImplSocket)graph.getSession();
  63         }
  64
  65         DebugSupportImpl() {
  66
  67                 getCommands.put("listeners", new FunctionImpl2<WriteGraph, String, Object>() {
  68
  69                         @Override
  70                         public Object apply(WriteGraph graph, String args) {
  71                                 try {
  72                                         return getSession(graph).queryProvider2.getListenerReport();
  73                                 } catch (IOException e) {
  74                                         Logger.defaultLogError(e);
  75                                         return e.getMessage();
  76                                 }
  77                         }
  78
  79                 });
  80
  81                 listCommands.put("counters", new FunctionImpl3<WriteGraph, File, String, String>() {
  82
  83                         @Override
  84                         public String apply(WriteGraph graph, File file, String args) {
  85                                 try {
  86                                         return ReadGraphImpl.listCounters(file);
  87                                 } catch (IOException e) {
  88                                         Logger.defaultLogError(e);
  89                                         return e.getMessage();
  90                                 }
  91                         }
  92
  93                 });
  94
  95                 listCommands.put("queries", new FunctionImpl3<WriteGraph, File, String, String>() {
  96
  97                         @Override
  98                         public String apply(WriteGraph graph, File file, String args) {
  99                                 try {
 100                                         return getSession(graph).queryProvider2.reportQueries(file);
 101                                 } catch (IOException e) {
 102                                         Logger.defaultLogError(e);
 103                                         return e.getMessage();
 104                                 }
 105                         }
 106
 107                 });
 108
 109                 listCommands.put("queryActivity", new FunctionImpl3<WriteGraph, File, String, String>() {
 110
 111                         @Override
 112                         public String apply(WriteGraph graph, File file, String args) {
 113                                 try {
 114                                         return getSession(graph).queryProvider2.reportQueryActivity(file);
 115                                 } catch (IOException e) {
 116                                         Logger.defaultLogError(e);
 117                                         return e.getMessage();
 118                                 }
 119                         }
 120
 121                 });
 122
 123                 listCommands.put("listeners", new FunctionImpl3<WriteGraph, File, String, String>() {
 124
 125                         @Override
 126                         public String apply(WriteGraph graph, File file, String args) {
 127                                 try {
 128                                         return getSession(graph).queryProvider2.reportListeners(file);
 129                                 } catch (IOException e) {
 130                                         Logger.defaultLogError(e);
 131                                         return e.getMessage();
 132                                 }
 133                         }
 134
 135                 });
 136
 137                 listCommands.put("clusters", new FunctionImpl3<WriteGraph, File, String, String>() {
 138
 139                         @Override
 140                         public String apply(WriteGraph graph, File file, String args) {
 141                                 return reportClusters(getSession(graph), file);
 142                         }
 143
 144                 });
 145
 146         listCommands.put("cluster", new FunctionImpl3<WriteGraph, File, String, String>() {
 147
 148             @Override
 149             public String apply(WriteGraph graph, File file, String args) {
 150                 return reportCluster(graph, file, args);
 151             }
 152
 153         });
 154
 155                 listCommands.put("virtuals", new FunctionImpl3<WriteGraph, File, String, String>() {
 156
 157                         @Override
 158                         public String apply(WriteGraph graph, File file, String args) {
 159                                 return reportVirtuals(getSession(graph), file);
 160                         }
 161
 162                 });
 163
 164                 listCommands.put("heap", new FunctionImpl3<WriteGraph, File, String, String>() {
 165
 166                         @Override
 167                         public String apply(WriteGraph graph, File file, String args) {
 168
 169                                 try {
 170
 171                                         file.delete();
 172
 173                                         Object bean = getBean();
 174                                         if (bean == null)
 175                                                 return "Could not retrieve bean.";
 176
 177                                         Method m = bean.getClass().getMethod("dumpHeap", String.class, boolean.class);
 178                                         if (args.length() > 0) {
 179                                                 m.invoke(bean, file.getParent() + "/" + args, true);
 180                                         } else {
 181                                                 m.invoke(bean, file.getAbsolutePath(), true);
 182                                         }
 183
 184                                 } catch (Throwable t) {
 185                                         Logger.defaultLogError(t);
 186                                         return "Unexpected exception " + t;
 187                                 }
 188
 189                                 return "Wrote " + file.getAbsolutePath();
 190
 191                     }
 192
 193                     private Object getBean() {
 194                         Class<?> beanClass = getBeanClass();
 195                         if (beanClass == null)
 196                             return null;
 197                         try {
 198                             Object bean = ManagementFactory.newPlatformMXBeanProxy(
 199                                     ManagementFactory.getPlatformMBeanServer(),
 200                                     "com.sun.management:type=HotSpotDiagnostic",
 201                                     beanClass);
 202                             return bean;
 203                         } catch (IOException e) {
 204                             return null;
 205                         }
 206                     }
 207
 208                     private Class<?> getBeanClass() {
 209                         try {
 210                             Class<?> clazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
 211                             return clazz;
 212                         } catch (ClassNotFoundException e) {
 213                             return null;
 214                         }
 215                     }
 216
 217                 });
 218
 219                 execCommands.put("WriteLogger.read", new FunctionImpl2<WriteGraph, String, String>() {
 220
 221                         @Override
 222                         public String apply(WriteGraph graph, String args) {
 223                                 graph.getSession().async(new WriteRequest() {
 224
 225                                         @Override
 226                                         public void perform(WriteGraph graph) throws DatabaseException {
 227                                                 try {
 228                                                         WriteLogger.read(graph);
 229                                                 } catch (Exception e) {
 230                                                         e.printStackTrace();
 231                                                 }
 232                                         }
 233
 234                                 });
 235                                 return "Started to read the write log.";
 236                         }
 237
 238                 });
 239
 240                 execCommands.put("ReadGraph.resetCounters", new FunctionImpl2<WriteGraph, String, String>() {
 241
 242                         @Override
 243                         public String apply(WriteGraph graph, String args) {
 244                                 graph.getSession().async(new WriteRequest() {
 245
 246                                         @Override
 247                                         public void perform(WriteGraph graph) throws DatabaseException {
 248                                                 try {
 249                                                         ReadGraphImpl.resetCounters();
 250                                                 } catch (Exception e) {
 251                                                         e.printStackTrace();
 252                                                 }
 253                                         }
 254
 255                                 });
 256                                 return "Started to read the write log.";
 257                         }
 258
 259                 });
 260
 261                 execCommands.put("QueryControl.flush", new FunctionImpl2<WriteGraph, String, String>() {
 262
 263                         @Override
 264                         public String apply(WriteGraph graph, String args) {
 265                                 QueryControl qc = graph.getService(QueryControl.class);
 266                                 qc.flush(graph);
 267                                 return "Flushed queries.";
 268                         }
 269
 270                 });
 271
 272         execCommands.put("DebugSupport.validateClusters", new FunctionImpl2<WriteGraph, String, String>() {
 273
 274             @Override
 275             public String apply(WriteGraph graph, String args) {
 276                 return validateClusters(graph);
 277             }
 278
 279         });
 280
 281         execCommands.put("DebugSupport.writeForMs", new FunctionImpl2<WriteGraph, String, String>() {
 282
 283             @Override
 284             public String apply(WriteGraph graph, String args) {
 285                 Integer amount = Integer.parseInt(args);
 286                 return writeForMs(graph, amount);
 287             }
 288
 289         });
 290
 291                 printCommands.put("nextId", new FunctionImpl2<WriteGraph, String, String>() {
 292
 293                         @Override
 294                         public String apply(WriteGraph graph, String args) {
 295                                 try {
 296                                         SessionImplSocket session = getSession(graph);
 297                                         ClusterImpl cluster = session.clusterTable.getNewResourceCluster(session.clusterTranslator, session.graphSession, false);
 298                                         long cid = cluster.getClusterId();
 299                                         int rid = cluster.getNumberOfResources(session.clusterTranslator);
 300                                         return cid + "#" + rid;
 301                                 } catch (Throwable t) {
 302                                         return UUID.randomUUID().toString();
 303                                 }
 304                         }
 305
 306                 });
 307
 308                 printCommands.put("usedMemory", new FunctionImpl2<WriteGraph, String, String>() {
 309
 310                         @Override
 311                         public String apply(WriteGraph graph, String args) {
 312                                 try {
 313                                         Runtime runtime = Runtime.getRuntime();
 314                                         return "" + (runtime.totalMemory() - runtime.freeMemory());
 315                                 } catch (Throwable t) {
 316                                         return UUID.randomUUID().toString();
 317                                 }
 318                         }
 319
 320                 });
 321
 322         }
 323
 324         private String reportClusters(SessionImplSocket session, File file) {
 325
 326                 try {
 327                         StringBuilder b = new StringBuilder();
 328                         long totalApproxSize = 0;
 329                         int loaded = 0;
 330                         for(ClusterI cluster : session.clusterTable.getClusters()) {
 331                                 b.append("[" + cluster.getClusterKey() + "]: ");
 332                                 if(cluster instanceof ClusterSmall) b.append("ClusterSmall[" + cluster.getClusterId() + "]");
 333                                 if(cluster instanceof ClusterBig) b.append("ClusterBig[" + cluster.getClusterId() + "]");
 334                                 if(cluster instanceof ClusterWriteOnly) b.append("ClusterWriteOnly[" + cluster.getClusterId() + "]");
 335                                 if(cluster.isLoaded()) {
 336                                         long approx = cluster.getUsedSpace();
 337                                         b.append(" approx size = " + approx + " bytes.\n");
 338                                         totalApproxSize += approx;
 339                                         loaded++;
 340                                 } else {
 341                                         b.append(" is not loaded.\n");
 342                                 }
 343                         }
 344                         b.append("#Total approx size is " + totalApproxSize + " bytes.\n");
 345                         b.append("#Amount of loaded clusters is " + loaded + ".\n");
 346                         FileUtils.writeFile(file, b.toString().getBytes());
 347                 } catch (IOException e) {
 348                         e.printStackTrace();
 349                 } catch (DatabaseException e) {
 350                         e.printStackTrace();
 351                 }
 352
 353                 return "OK";
 354
 355         }
 356
 357     private String reportCluster(final WriteGraph graph, File file, String args) {
 358
 359         try {
 360
 361             final StringBuilder b = new StringBuilder();
 362             final SessionImplSocket session = (SessionImplSocket)graph.getSession();
 363             long clusterId = Long.parseLong(args);
 364             b.append("cluster id: " + clusterId);
 365             b.append("\n");
 366             b.append("internal resources: ");
 367             b.append("\n");
 368             ClusterI cluster = session.clusterTable.getClusterByClusterId(clusterId);
 369             for(int i=1;i<=cluster.getNumberOfResources(session.clusterTranslator);i++) {
 370                 Resource r = session.getResource(i, clusterId);
 371                 String def = NameUtils.getSafeName(graph, r);
 372                 b.append( i+": " + def);
 373                 b.append("\n");
 374             }
 375             if(cluster instanceof ClusterSmall) {
 376                 ClusterSmall clusterSmall = (ClusterSmall)cluster;
 377                 b.append("foreign resources: ");
 378                 b.append("\n");
 379                 final TIntIntHashMap clusterHistogram = new TIntIntHashMap();
 380                 clusterSmall.foreignTable.getResourceHashMap().forEachEntry(new TIntShortProcedure() {
 381
 382                     @Override
 383                     public boolean execute(int index, short pos) {
 384                         try {
 385                             Resource r = session.getResource(index);
 386                             String def = NameUtils.getSafeName(graph, r, true);
 387                             int cluster = index >>> 12;
 388                             int key = index & 0xfff;
 389                             int exist = clusterHistogram.get(cluster);
 390                             clusterHistogram.put(cluster, exist+1);
 391                             b.append( cluster + "$" + key +": " + def);
 392                             b.append("\n");
 393                         } catch (ValidationException e) {
 394                             e.printStackTrace();
 395                         } catch (ServiceException e) {
 396                             e.printStackTrace();
 397                         }
 398                         return true;
 399                     }
 400                 });
 401                 b.append("foreign histogram: ");
 402                 b.append("\n");
 403                 clusterHistogram.forEachEntry(new TIntIntProcedure() {
 404
 405                     @Override
 406                     public boolean execute(int cluster, int count) {
 407                         b.append( cluster +": " + count);
 408                         b.append("\n");
 409                         return true;
 410                     }
 411                 });
 412             }
 413
 414             FileUtils.writeFile(file, b.toString().getBytes());
 415         } catch (IOException e) {
 416             e.printStackTrace();
 417         } catch (DatabaseException e) {
 418             e.printStackTrace();
 419         }
 420
 421         return "OK";
 422
 423     }
 424
 425         private String reportVirtuals(SessionImplSocket session, File file) {
 426
 427                 session.virtualGraphServerSupport.report(file);
 428
 429                 return "OK";
 430
 431         }
 432
 433         public Object query(Session session, final String command) {
 434                 try {
 435                         return session.sync(new WriteResultRequest<Object>() {
 436
 437                                 @Override
 438                                 public Object perform(WriteGraph graph) throws DatabaseException {
 439                                         return query(graph, command);
 440                                 }
 441
 442                         });
 443                 } catch (DatabaseException e) {
 444                         Logger.defaultLogError(e);
 445                         return null;
 446                 }
 447         }
 448
 449         @Override
 450         public Object query(WriteGraph graph, String command) {
 451
 452                 Bundle bundle = Platform.getBundle(Activator.BUNDLE_ID);
 453                 File base = Utils.getBaseFile(bundle);
 454
 455                 command = command.trim();
 456
 457                 try {
 458
 459                         if("help".equals(command)) {
 460
 461                                 return
 462                                                 "Welcome to the Simantics session debugger.<br><br>" +
 463                                                 "This shell allows you to make following queries into the running Simantics database session:<br>" +
 464                                                 "<ul><li>Get commands, which return debug objects. Type 'help get' to obtain more information.</li>" +
 465                                                 "<li>List commands, which create debug listings into files. Type 'help list' to obtain more information.</li>" +
 466                                                 "<li>Print commands, which output information about session state. Type 'help print' to obtain more information.</li>" +
 467                                                 "<li>Set commands, which modify session state variables. Type 'help set' to obtain more information.</li>" +
 468                                                 "<li>Exec commands, which perform certain actions. Type 'help exec' to obtain more information.</li></ul>"
 469                                                 ;
 470
 471                         } else if ("help get".equals(command)) {
 472
 473                                 StringBuilder b = new StringBuilder();
 474                                 b.append("The following get commands are available.<br><ul>");
 475                                 for(String key : getCommands.keySet())
 476                                         b.append("<li>" + key + "</li>");
 477
 478                                 b.append("</ul>");
 479
 480                                 return b.toString();
 481
 482                         } else if ("help list".equals(command)) {
 483
 484                                 StringBuilder b = new StringBuilder();
 485                                 b.append("The following list commands are available.<br><ul>");
 486                                 for(String key : listCommands.keySet())
 487                                         b.append("<li>" + key + "</li>");
 488
 489                                 b.append("</ul>");
 490
 491                                 return b.toString();
 492
 493                         } else if ("help exec".equals(command)) {
 494
 495                                 StringBuilder b = new StringBuilder();
 496                                 b.append("The following exec commands are available.<br><ul>");
 497                                 for(String key : execCommands.keySet())
 498                                         b.append("<li>" + key + "</li>");
 499
 500                                 b.append("</ul>");
 501
 502                                 return b.toString();
 503
 504                         } else if ("help print".equals(command)) {
 505
 506                                 StringBuilder b = new StringBuilder();
 507                                 b.append("The following print commands are available.<br><ul>");
 508                                 for(String key : printCommands.keySet())
 509                                         b.append("<li>" + key + "</li>");
 510
 511                                 b.append("</ul>");
 512
 513                                 return b.toString();
 514
 515                         } else if ("help set".equals(command)) {
 516
 517                                 StringBuilder b = new StringBuilder();
 518                                 b.append("The following set commands are available.<br><ul>");
 519
 520                                 for(Map.Entry<String, Variant> e : Development.getProperties().entrySet()) {
 521                                         b.append("<li>" + e.getKey() + " - " + e.getValue().getBinding().type() + "</li>");
 522                                 }
 523
 524                                 b.append("</ul>");
 525
 526                                 return b.toString();
 527
 528                         } else if(command.startsWith("get")) {
 529
 530                                 String remainder = command.substring(3).trim();
 531
 532                                 for(Map.Entry<String, Function2<WriteGraph, String, Object>> e : getCommands.entrySet()) {
 533                                         String key = e.getKey();
 534                                         if(remainder.startsWith(key)) {
 535                                                 String args = remainder.substring(key.length()).trim();
 536                                                 return e.getValue().apply(graph, args);
 537                                         }
 538                                 }
 539
 540                         } else if(command.startsWith("list")) {
 541
 542                                 String remainder = command.substring(4).trim();
 543
 544                                 for(Map.Entry<String, Function3<WriteGraph, File, String, String>> e : listCommands.entrySet()) {
 545                                         String key = e.getKey();
 546                                         if(remainder.startsWith(key)) {
 547                                                 String args = remainder.substring(key.length()).trim();
 548                                                 File file = new File(base, key + ".list");
 549                                                 base.mkdirs();
 550                                                 e.getValue().apply(graph, file, args);
 551                                                 return "Wrote " + file.getAbsolutePath();
 552                                         }
 553                                 }
 554
 555                         } else if(command.startsWith("set")) {
 556
 557                                 String remainder = command.substring(3).trim();
 558                                 String[] keyAndValue = remainder.split("=");
 559                                 if(keyAndValue.length == 2) {
 560
 561                                         Variant exist = Development.getProperties().get(keyAndValue[0]);
 562                                         if(exist != null) {
 563                                                 Development.setProperty(keyAndValue[0], exist.getBinding().parseValue(keyAndValue[1], new DataValueRepository()), exist.getBinding());
 564                                                 return "Property " + keyAndValue[0] + " was set to '" + keyAndValue[1] + "'";
 565                                         } else {
 566                                                 return query(graph, "help set");
 567                                         }
 568                                 } else {
 569                                         return query(graph, "help set");
 570                                 }
 571
 572                         } else if(command.startsWith("print")) {
 573
 574                                 String remainder = command.substring(5).trim();
 575
 576                                 for(Map.Entry<String, Function2<WriteGraph, String, String>> e : printCommands.entrySet()) {
 577                                         String key = e.getKey();
 578                                         if(remainder.startsWith(key)) {
 579                                                 String args = remainder.substring(key.length()).trim();
 580                                                 return e.getValue().apply(graph, args);
 581                                         }
 582                                 }
 583
 584                         } else if(command.startsWith("exec")) {
 585
 586                                 String remainder = command.substring(4).trim();
 587
 588                                 for(Map.Entry<String, Function2<WriteGraph, String, String>> e : execCommands.entrySet()) {
 589                                         String key = e.getKey();
 590                                         if(remainder.startsWith(key)) {
 591                                                 String args = remainder.substring(key.length()).trim();
 592                                                 return e.getValue().apply(graph, args);
 593                                         }
 594                                 }
 595
 596                         }
 597
 598                         return "Unknown command '" + command + "'";
 599
 600                 } catch (Throwable t) {
 601
 602                         t.printStackTrace();
 603
 604                         return t.getMessage();
 605
 606                 }
 607
 608         }
 609         private void writeResouce(WriteGraph graph) throws DatabaseException {
 610         Layer0 l0 = Layer0.getInstance(graph);
 611         Resource r = graph.newResource();
 612         graph.claim(r, l0.InstanceOf, l0.Entity);
 613         XSupport xs = (XSupport)graph.getService(XSupport.class);
 614         xs.flushCluster(r);
 615         }
 616         private void wait(int amount) {
 617         long start = System.nanoTime();
 618         long limit = amount * 1000000L;
 619         while ((System.nanoTime() - start) < limit) {
 620             try {
 621                 Thread.sleep(100);
 622             } catch (InterruptedException e) {
 623                 e.printStackTrace();
 624             }
 625         }
 626         }
 627         String writeForMs(WriteGraph graph, int amount) {
 628         try {
 629             writeResouce(graph);
 630             wait(amount);
 631             writeResouce(graph);
 632         } catch (DatabaseException e) {
 633             e.printStackTrace();
 634         }
 635
 636                 return "Slept for " + amount + " ms in write transaction.";
 637         }
 638
 639         String validateClusters(WriteGraph graph) {
 640
 641             ReadGraphImpl impl = (ReadGraphImpl)graph;
 642             QueryProcessor processor = impl.processor;
 643
 644             QuerySupport qs = graph.getService(QuerySupport.class);
 645
 646             TIntHashSet done = new TIntHashSet();
 647             TreeSet<Integer> fringe = new TreeSet<Integer>();
 648
 649             ResourceImpl root = (ResourceImpl)graph.getRootLibrary();
 650
 651             done.add(root.id);
 652             fringe.add(root.id);
 653
 654             while(!fringe.isEmpty()) {
 655                 int r = fringe.first();
 656                 fringe.remove(r);
 657                 DirectStatements ds = qs.getStatements(impl, r, processor, true);
 658             for(Statement stm : ds) {
 659                 ResourceImpl p = (ResourceImpl)stm.getPredicate();
 660                 ResourceImpl o = (ResourceImpl)stm.getObject();
 661                 if(done.add(p.id)) fringe.add(p.id);
 662                 if(done.add(o.id)) fringe.add(o.id);
 663             }
 664                 qs.getValue(impl, r);
 665             }
 666
 667             return "Validated " + done.size() + " resources.";
 668
 669         }
 670
 671 }