private void includeCodes(ConceptSetComponent inc, List<ValueSetExpansionParameterComponent> params, Parameters expParams, boolean heirarchical) throws ETooCostly, FileNotFoundException, IOException, FHIRException {
  inc.checkNoModifiers("Compose.include", "expanding");
  List<ValueSet> imports = new ArrayList<ValueSet>();
  for (UriType imp : inc.getValueSet()) {
    imports.add(importValueSet(imp.getValue(), params, expParams));
  }

  if (!inc.hasSystem()) {
    if (imports.isEmpty()) // though this is not supposed to be the case
      return;
    ValueSet base = imports.get(0);
    imports.remove(0);
    base.checkNoModifiers("Imported ValueSet", "expanding");
    copyImportContains(base.getExpansion().getContains(), null, expParams, imports);
  } else {
    CodeSystem cs = context.fetchCodeSystem(inc.getSystem());
    if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE)) {
      doServerIncludeCodes(inc, heirarchical, params, imports, expParams);
    } else {
      doInternalIncludeCodes(inc, params, expParams, imports, cs);
    }
  }
}
 

private void doServerIncludeCodes(ConceptSetComponent inc, boolean heirarchical, List<ValueSetExpansionParameterComponent> params, List<ValueSet> imports, Parameters expParams) throws FHIRException {
  ValueSetExpansionOutcome vso = context.expandVS(inc, heirarchical);
  if (vso.getError() != null)
    throw new TerminologyServiceException("Unable to expand imported value set: " + vso.getError());
  ValueSet vs = vso.getValueset();
  if (vs.hasVersion())
    if (!existsInParams(params, "version", new UriType(vs.getUrl() + "|" + vs.getVersion())))
      params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(vs.getUrl() + "|" + vs.getVersion())));
  for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) {
    if (!existsInParams(params, p.getName(), p.getValue()))
      params.add(p);
  }
  for (ValueSetExpansionContainsComponent cc : vs.getExpansion().getContains()) {
    addCodeAndDescendents(cc, null, expParams, imports);
  }
}
 

private ValueSet intersectByExpansion(ValueSet lvs, ValueSet rvs) {
  // this is pretty straight forward - we intersect the lists, and build a compose out of the intersection
  ValueSet vs = new ValueSet();
  vs.setStatus(PublicationStatus.DRAFT);
  
  Map<String, ValueSetExpansionContainsComponent> left = new HashMap<String, ValueSetExpansionContainsComponent>();
  scan(lvs.getExpansion().getContains(), left);
  Map<String, ValueSetExpansionContainsComponent> right = new HashMap<String, ValueSetExpansionContainsComponent>();
  scan(rvs.getExpansion().getContains(), right);
  Map<String, ConceptSetComponent> inc = new HashMap<String, ConceptSetComponent>();
  
  for (String s : left.keySet()) {
    if (right.containsKey(s)) {
      ValueSetExpansionContainsComponent cc = left.get(s);
      ConceptSetComponent c = inc.get(cc.getSystem());
      if (c == null) {
        c = vs.getCompose().addInclude().setSystem(cc.getSystem());
        inc.put(cc.getSystem(), c);
      }
      c.addConcept().setCode(cc.getCode()).setDisplay(cc.getDisplay());
    }
  }
  return vs;
}
 

private String getIncSummary(ConceptSetComponent cc) {
  CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
  for (UriType vs : cc.getValueSet())
    b.append(vs.asStringValue());
  String vsd = b.length() > 0 ? " where the codes are in the value sets ("+b.toString()+")" : "";
  String system = cc.getSystem();
  if (cc.hasConcept())
    return Integer.toString(cc.getConcept().size())+" codes from "+system+vsd;
  if (cc.hasFilter()) {
    String s = "";
    for (ConceptSetFilterComponent f : cc.getFilter()) {
      if (!Utilities.noString(s))
        s = s + " & ";
      s = s + f.getProperty()+" "+f.getOp().toCode()+" "+f.getValue();
    }
    return "from "+system+" where "+s+vsd;
  }
  return "All codes from "+system+vsd;
}
 

private void makeFullVs(String dest, CodeSystem cs) throws FileNotFoundException, IOException {
  String url = "http://id.who.int/icd11/ValueSet/all-MMS";
  ValueSet vs = new ValueSet();
  vs.setId("all-MMS");
  vs.setUrl(url);
  vs.setName("ICDMMSAll");
  vs.setTitle("Value Set for all ICD MMS Codes");
  vs.setStatus(PublicationStatus.ACTIVE);
  vs.setExperimental(false);
  vs.setDate(cs.getDate());
  vs.setPublisher("WHO");
  vs.setCopyright("Consult WHO For terms of use");
  vs.setVersion(cs.getVersion());
  vs.setStatus(cs.getStatus());
  ConceptSetComponent inc = vs.getCompose().addInclude();
  inc.setSystem(cs.getUrl());
  new XmlParser(XmlVersion.V1_1).setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "vs-all-MMS.xml")), vs);
}
 

private void makeFullVs2(String dest, CodeSystem cs) throws FileNotFoundException, IOException {
  String url = "http://id.who.int/icd11/ValueSet/all-foundation";
  ValueSet vs = new ValueSet();
  vs.setId("all-foundation");
  vs.setUrl(url);
  vs.setName("ICDFoundationAll");
  vs.setTitle("Value Set for all ICD Foundation Concepts");
  vs.setStatus(PublicationStatus.ACTIVE);
  vs.setExperimental(false);
  vs.setDate(cs.getDate());
  vs.setPublisher("WHO");
  vs.setCopyright("Consult WHO For terms of use");
  vs.setVersion(cs.getVersion());
  vs.setStatus(cs.getStatus());
  ConceptSetComponent inc = vs.getCompose().addInclude();
  inc.setSystem(cs.getUrl());
  new XmlParser(XmlVersion.V1_1).setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "vs-all-foundation.xml")), vs);
}
 

private String buildValueSet(CodeSystem cs, String code, String name, JsonObject o, String dest) throws FileNotFoundException, IOException {
  String id = code+"-"+name;
  String url = "http://id.who.int/icd11/ValueSet/"+id;
  ValueSet vs = new ValueSet();
  vs.setId(id);
  vs.setUrl(url);
  vs.setName("VS"+name+"4"+code);
  vs.setTitle("Value Set for "+name+" on "+code);
  vs.setStatus(PublicationStatus.ACTIVE);
  vs.setExperimental(false);
  vs.setDate(cs.getDate());
  vs.setPublisher("WHO");
  vs.setCopyright("Consult WHO For terms of use");
  vs.setVersion(cs.getVersion());
  vs.setStatus(cs.getStatus());
  ConceptSetComponent inc = vs.getCompose().addInclude();
  inc.setSystem(cs.getUrl());
  for (JsonElement e : o.getAsJsonArray("scaleEntity")) {
    inc.addFilter().setProperty("concept").setOp(FilterOperator.ISA).setValue(tail(e.getAsString()));
  }    
  new XmlParser(XmlVersion.V1_1).setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "vs-"+id+".xml")), vs);
  return url;
}
 

private static void checkValueSet(ValueSet valueSet, String url, String version) {

    Assert.assertNotNull(
        MessageFormat.format("Could not find value set for url {0} and version {1}", url, version),
        valueSet);

    Assert.assertEquals(url, valueSet.getUrl());
    Assert.assertEquals(version, valueSet.getVersion());

    ConceptSetComponent inclusion = valueSet.getCompose().getIncludeFirstRep();

    Assert.assertEquals("urn:cerner:system", inclusion.getSystem());
    Assert.assertEquals("1", inclusion.getVersion());
    Assert.assertEquals("a", inclusion.getConceptFirstRep().getCode());

    Assert.assertEquals(1, valueSet.getCompose().getInclude().size());
  }
 

private void excludeCodes(ConceptSetComponent exc, List<ValueSetExpansionParameterComponent> params, String ctxt) throws FHIRException {
  exc.checkNoModifiers("Compose.exclude", "expanding");
  if (exc.hasSystem() && exc.getConcept().size() == 0 && exc.getFilter().size() == 0) {
    excludeSystems.add(exc.getSystem());
  }

  if (exc.hasValueSet())
    throw new Error("Processing Value set references in exclude is not yet done in "+ctxt);
  // importValueSet(imp.getValue(), params, expParams);

  CodeSystem cs = context.fetchCodeSystem(exc.getSystem());
  if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(exc.getSystem())) {
    ValueSetExpansionOutcome vse = context.expandVS(exc, false);
    ValueSet valueset = vse.getValueset();
    if (valueset == null)
      throw new TerminologyServiceException("Error Expanding ValueSet: "+vse.getError());
    excludeCodes(valueset.getExpansion(), params);
    return;
  }

  for (ConceptReferenceComponent c : exc.getConcept()) {
    excludeCode(exc.getSystem(), c.getCode());
  }

  if (exc.getFilter().size() > 0)
    throw new NotImplementedException("not done yet");
}
 

private boolean mergeIntoExisting(List<ConceptSetComponent> include, ConceptSetComponent inc) {
  for (ConceptSetComponent dst : include) {
    if (Base.compareDeep(dst,  inc, false))
      return true; // they're actually the same
    if (dst.getSystem().equals(inc.getSystem())) {
      if (inc.hasFilter() || dst.hasFilter()) {
        return false; // just add the new one as a a parallel
      } else if (inc.hasConcept() && dst.hasConcept()) {
        for (ConceptReferenceComponent cc : inc.getConcept()) {
          boolean found = false;
          for (ConceptReferenceComponent dd : dst.getConcept()) {
            if (dd.getCode().equals(cc.getCode()))
              found = true;
            if (found) {
              if (cc.hasDisplay() && !dd.hasDisplay())
                dd.setDisplay(cc.getDisplay());
              break;
            }
          }
          if (!found)
            dst.getConcept().add(cc.copy());
        }
      } else
        dst.getConcept().clear(); // one of them includes the entire code system 
    }
  }
  return false;
}
 

@Override
public ValueSetExpansionOutcome expandVS(ConceptSetComponent inc, boolean heirachical) throws TerminologyServiceException {
  ValueSet vs = new ValueSet();
  vs.setCompose(new ValueSetComposeComponent());
  vs.getCompose().getInclude().add(inc);
  CacheToken cacheToken = txCache.generateExpandToken(vs, heirachical);
  ValueSetExpansionOutcome res;
  res = txCache.getExpansion(cacheToken);
  if (res != null)
    return res;
  Parameters p = expParameters.copy(); 
  p.setParameter("includeDefinition", false);
  p.setParameter("excludeNested", !heirachical);
  
  if (noTerminologyServer)
    return new ValueSetExpansionOutcome("Error expanding ValueSet: running without terminology services", TerminologyServiceErrorClass.NOSERVICE);
  Map<String, String> params = new HashMap<String, String>();
  params.put("_limit", Integer.toString(expandCodesLimit ));
  params.put("_incomplete", "true");
  tlog("$expand on "+txCache.summary(vs));
  try {
    ValueSet result = txClient.expandValueset(vs, p, params);
    res = new ValueSetExpansionOutcome(result).setTxLink(txLog.getLastId());  
  } catch (Exception e) {
    res = new ValueSetExpansionOutcome(e.getMessage() == null ? e.getClass().getName() : e.getMessage(), TerminologyServiceErrorClass.UNKNOWN);
    if (txLog != null)
      res.setTxLink(txLog.getLastId());
  }
  txCache.cacheExpansion(cacheToken, res, TerminologyCache.PERMANENT);
  return res;



}
 

@Override
public ValidationResult validateCode(ValidationOptions options, String system, String code, String display, ConceptSetComponent vsi) {
  Coding c = new Coding(system, code, display);
  ValueSet vs = new ValueSet();
  vs.setUrl(Utilities.makeUuidUrn());
  vs.getCompose().addInclude(vsi);
  return validateCode(options, c, vs);
}
 

/**
 * Returns a new ValueSets instance that includes the given value sets.
 *
 * @param valueSets the value sets to add to the returned collection.
 * @return a new ValueSets instance with the added value sets.
 */
@Override
public ValueSets withValueSets(Dataset<ValueSet> valueSets) {

  Dataset<UrlAndVersion> newMembers = getUrlAndVersions(valueSets);

  // Ensure that there are no duplicates among the value sets
  if (hasDuplicateUrlAndVersions(newMembers) || valueSets.count() != newMembers.count()) {

    throw new IllegalArgumentException(
        "Cannot add value sets having duplicate valueSetUri and valueSetVersion");
  }

  // The value set concepts will be stored in the values table for persistence, so we remove
  // them from the individual value sets. This can be done most easily by setting concepts to an
  // empty list.
  Dataset<ValueSet> withoutConcepts = valueSets.map((MapFunction<ValueSet,ValueSet>) valueSet -> {
    ValueSet valueSetWithoutConcepts = valueSet.copy();

    List<ConceptSetComponent> updatedInclusions = new ArrayList<>();

    for (ConceptSetComponent inclusion: valueSet.getCompose().getInclude()) {

      ConceptSetComponent inclusionWithoutConcepts = inclusion.copy();

      inclusionWithoutConcepts.setConcept(new ArrayList<>());
      updatedInclusions.add(inclusionWithoutConcepts);
    }

    valueSetWithoutConcepts.getCompose().setInclude(updatedInclusions);

    return valueSetWithoutConcepts;
  }, VALUE_SET_ENCODER);

  Dataset<Value> newValues = valueSets.flatMap(ValueSets::expandValuesIterator,
      getValueEncoder());

  return withValueSets(withoutConcepts, newValues);
}
 

private void addConcepts(ConceptSetComponent theInclude, ValueSetExpansionComponent theRetVal, Set<String> theWantCodes, List<ConceptDefinitionComponent> theConcepts) {
    for (ConceptDefinitionComponent next : theConcepts) {
        if (theWantCodes.isEmpty() || theWantCodes.contains(next.getCode())) {
            theRetVal
                    .addContains()
                    .setSystem(theInclude.getSystem())
                    .setCode(next.getCode())
                    .setDisplay(next.getDisplay());
        }
        addConcepts(theInclude, theRetVal, theWantCodes, next.getConcept());
    }
}
 

@Override
public ValueSetExpansionOutcome expandValueSet(FhirContext theContext,
    ConceptSetComponent theInclude) {
  if (theInclude.getSystem().equals("urn:iso:std:iso:4217")) {
    ValueSet valueset =  (ValueSet) resourcesMap.get("http://hl7.org/fhir/ValueSet/currencies");
    ValueSetExpansionOutcome expansion = new ValueSetExpansionOutcome(valueset);
    return expansion;
  }
  return null;
}
 

public static void markStatus(ValueSet vs, String wg, StandardsStatus status, String pckage, String fmm, IWorkerContext context, String normativeVersion) throws FHIRException {
  if (vs.hasUserData("external.url"))
    return;
  
  if (wg != null) {
    if (!ToolingExtensions.hasExtension(vs, ToolingExtensions.EXT_WORKGROUP) || 
        (!Utilities.existsInList(ToolingExtensions.readStringExtension(vs, ToolingExtensions.EXT_WORKGROUP), "fhir", "vocab") && Utilities.existsInList(wg, "fhir", "vocab"))) {
      ToolingExtensions.setCodeExtension(vs, ToolingExtensions.EXT_WORKGROUP, wg);
    }
  }
  if (status != null) {
    StandardsStatus ss = ToolingExtensions.getStandardsStatus(vs);
    if (ss == null || ss.isLowerThan(status)) 
      ToolingExtensions.setStandardsStatus(vs, status, normativeVersion);
    if (pckage != null) {
      if (!vs.hasUserData("ballot.package"))        
        vs.setUserData("ballot.package", pckage);
      else if (!pckage.equals(vs.getUserString("ballot.package")))
        if (!"infrastructure".equals(vs.getUserString("ballot.package")))
        System.out.println("Value Set "+vs.getUrl()+": ownership clash "+pckage+" vs "+vs.getUserString("ballot.package"));
    }
    if (status == StandardsStatus.NORMATIVE) {
      vs.setExperimental(false);
      vs.setStatus(PublicationStatus.ACTIVE);
    }
  }
  if (fmm != null) {
    String sfmm = ToolingExtensions.readStringExtension(vs, ToolingExtensions.EXT_FMM_LEVEL);
    if (Utilities.noString(sfmm) || Integer.parseInt(sfmm) < Integer.parseInt(fmm)) 
      ToolingExtensions.setIntegerExtension(vs, ToolingExtensions.EXT_FMM_LEVEL, Integer.parseInt(fmm));
  }
  if (vs.hasUserData("cs"))
    CodeSystemUtilities.markStatus((CodeSystem) vs.getUserData("cs"), wg, status, pckage, fmm, normativeVersion);
  else if (status == StandardsStatus.NORMATIVE && context != null) {
    for (ConceptSetComponent csc : vs.getCompose().getInclude()) {
      if (csc.hasSystem()) {
        CodeSystem cs = context.fetchCodeSystem(csc.getSystem());
        if (cs != null) {
          CodeSystemUtilities.markStatus(cs, wg, status, pckage, fmm, normativeVersion);
        }
      }
    }
  }
}
 

public static void addVSComment(ConceptSetComponent nc, String comment) {
  if (!StringUtils.isBlank(comment))
    nc.getExtension().add(Factory.newExtension(EXT_VS_COMMENT, Factory.newString_(comment), true));   
}
 

private static Iterator<Value> expandValuesIterator(ValueSet valueSet) {

    List<Value> values = new ArrayList<>();

    ValueSetComposeComponent compose = valueSet.getCompose();

    for (ConceptSetComponent inclusion: compose.getInclude()) {

      for (ConceptReferenceComponent concept: inclusion.getConcept()) {

        Value value = new Value();

        value.setValueSetUri(valueSet.getUrl());
        value.setValueSetVersion(valueSet.getVersion());

        value.setSystem(inclusion.getSystem());
        value.setVersion(inclusion.getVersion());

        value.setValue(concept.getCode());

        values.add(value);
      }
    }

    return values.iterator();
  }
 

@Override
protected void addToValueSet(ValueSet valueSet, Dataset<Value> values) {

  ValueSetComposeComponent composeComponent = valueSet.getCompose();
  ConceptSetComponent currentInclusion = null;
  ConceptReferenceComponent concept = null;

  List<Value> sortedValues = values.sort("system", "version", "value").collectAsList();

  // Workaround for the decoder producing an immutable array by replacing it with a mutable one
  composeComponent.setInclude(new ArrayList<>(composeComponent.getInclude()));
  for (Value value: sortedValues) {

    if (currentInclusion == null
        || !value.getSystem().equals(currentInclusion.getSystem())
        || !value.getVersion().equals(currentInclusion.getVersion())) {

      // Find a matching inclusion
      for (ConceptSetComponent candidate: composeComponent.getInclude()) {

        if (value.getSystem().equals(candidate.getSystem())
            && value.getVersion().equals(candidate.getVersion())) {

          currentInclusion = candidate;

          // Workaround for the decoder producing an immutable array by replacing it with a
          // mutable one
          currentInclusion.setConcept(new ArrayList<>(currentInclusion.getConcept()));
        }
      }

      // No matching inclusion found, so add one
      if (currentInclusion == null) {

        currentInclusion = composeComponent.addInclude();

        currentInclusion.setSystem(value.getSystem());
        currentInclusion.setVersion(value.getVersion());

        concept = null;
      }
    }

    // Create concept if not exists
    if (concept == null || !value.getValue().equals(concept.getCode())) {

      concept = currentInclusion.addConcept();
      concept.setCode(value.getValue());
    }
  }
}
 

@Override
public CodeValidationResult validateCode(FhirContext theContext, String theCodeSystem,
    String theCode, String theDisplay, String theValueSetUrl) {
  IssueSeverity severity = IssueSeverity.WARNING;
  String message = "Unsupported CodeSystem";

  if (isCodeSystemSupported(theContext, theCodeSystem)) {
    severity = IssueSeverity.ERROR;
    message = "Code not found";

    CodeSystem cs = codeSystemMap.get(theCodeSystem);
    for (ConceptDefinitionComponent def : cs.getConcept()) {
      if (def.getCode().equals(theCode)) {
        if (def.getDisplay() != null && theDisplay != null) {
          if (def.getDisplay().equals(theDisplay)) {
            severity = IssueSeverity.INFORMATION;
            message = "Validated Successfully";
          } else {
            severity = IssueSeverity.WARNING;
            message = "Validated Code; Display mismatch";
          }
        } else {
          severity = IssueSeverity.WARNING;
          message = "Validated Code; No display";
        }
      }
    }
  }

  ValueSet vs = fetchValueSet(theContext, theValueSetUrl);
  if (vs != null && vs.hasCompose() && vs.getCompose().hasExclude()) {
    for (ConceptSetComponent exclude : vs.getCompose().getExclude()) {
      if (exclude.getSystem().equals(theCodeSystem) && exclude.hasConcept()) {
        for (ConceptReferenceComponent concept : exclude.getConcept()) {
          if (concept.getCode().equals(theCode)) {
            severity = IssueSeverity.ERROR;
            message += "; Code Excluded from ValueSet";
          }
        }
      }
    }
  }

  return new CodeValidationResult(severity, message);
}
 

/**
 * Expand the value set fragment (system | codes | filters). Note that this 
 * might fail if the expansion is very large. If the expansion fails, then the 
 * checkVS will be called instead
 */
public ValueSetExpansionComponent expandVS(ConceptSetComponent inc) throws Exception;
 

/**
 * Test the value set fragment (system | codes | filters). 
 */
public boolean checkVS(ConceptSetComponent vsi, String system, String code);
 

/**
 * Value set expanion inside the internal expansion engine - used 
 * for references to supported system (see "supportsSystem") for
 * which there is no value set. 
 * 
 * @param inc
 * @return
 * @throws FHIRException 
 */
public ValueSetExpansionOutcome expandVS(ConceptSetComponent inc, boolean heirarchical) throws TerminologyServiceException;
 

/**
 * Validation of a code - consult the terminology service 
 * to see whether it is known. If known, return a description of it
 * Also, check whether it's in the provided value set fragment (for supported systems with no value set definition)
 * 
 * note: always return a result, with either an error or a code description, or both (e.g. known code, but not in the value set)
 *  
 * corresponds to 2 terminology service calls: $validate-code and $lookup
 * 
 * @param system
 * @param code
 * @param display
 * @return
 */
public ValidationResult validateCode(ValidationOptions options, String system, String code, String display, ConceptSetComponent vsi);