Merge pull request #10080 from gem/damages2

Honoring `total_losses` in damage calculations too

Author: micheles

openquake/calculators/event_based_damage.py

@@ -79,8 +79,8 @@ def damage_from_gmfs(gmfslices, oqparam, dstore, monitor):
     return event_based_damage(df, oqparam, dstore, monitor)
 
 
-def _gen_d4(asset_df, gmf_df, crmodel, dparam):
-    # yields (aids, d4) triples
+def _gen_d3(asset_df, gmf_df, crmodel, dparam):
+    # yields (aids, d3) triples
     oq = crmodel.oqparam
     sec_sims = oq.secondary_simulations.items()
     for prob_field, num_sims in sec_sims:
@@ -124,11 +124,18 @@ def _gen_d4(asset_df, gmf_df, crmodel, dparam):
                             d4[lti, a, :, d] *= dprobs
 
         df = crmodel.tmap_df[crmodel.tmap_df.taxi == assets[0]['taxonomy']]
+        if 'losses' in crmodel.get_consequences():
+            loss_types = oq.total_loss_types
+        else:
+            loss_types = {lt: i for i, lt in enumerate(oq.loss_types)}
         csq = crmodel.compute_csq(
-            assets, d4[:, :, :, :D], df, oq.loss_types, oq.time_event)
+            assets, d4[:, :, :, :D], df, loss_types, oq.time_event)
+        d3 = numpy.zeros((A, E, dparam.Dc), F32)
+        for li, lt in enumerate(oq.loss_types):
+            d3[:] += d4[li]
         for name, values in csq.items():
-            d4[:, :, :, dparam.csqidx[name]] = values
-        yield aids, d4  # d4 has shape (L, A, E, Dc)
+            d3[:, :, dparam.csqidx[name]] = values
+        yield aids, d3  # d3 has shape (A, E, Dc)
 
 
 def event_based_damage(df, oq, dstore, monitor):
@@ -176,28 +183,30 @@ def event_based_damage(df, oq, dstore, monitor):
             else:
                 rng = None
             dparam = Dparam(eids, aggids, rlzs, csqidx, D, Dc, rng)
-            for aids, d4 in _gen_d4(asset_df, gmf_df, crmodel, dparam):
-                for lti, d3 in enumerate(d4):
-                    if R == 1:
-                        dmgcsq[aids, 0] += d3.sum(axis=1)
-                    else:
-                        for e, rlz in enumerate(dparam.rlzs):
-                            dmgcsq[aids, rlz] += d3[:, e]
-                    tot = d3.sum(axis=0)  # sum on the assets
-                    for e, eid in enumerate(eids):
-                        dddict[eid, oq.K] += tot[e]
-                        if oq.K:
-                            for kids in dparam.aggids:
-                                for a, aid in enumerate(aids):
-                                    dddict[eid, kids[aid]] += d3[a, e]
-
-    return _dframe(dddict, csqidx, oq.loss_types), dmgcsq
+            for aids, d3 in _gen_d3(asset_df, gmf_df, crmodel, dparam):
+                if R == 1:
+                    dmgcsq[aids, 0] += d3.sum(axis=1)
+                else:
+                    for e, rlz in enumerate(dparam.rlzs):
+                        dmgcsq[aids, rlz] += d3[:, e]
+                tot = d3.sum(axis=0)  # sum on the assets
+                for e, eid in enumerate(eids):
+                    dddict[eid, oq.K] += tot[e]
+                    if oq.K:
+                        for kids in dparam.aggids:
+                            for a, aid in enumerate(aids):
+                                dddict[eid, kids[aid]] += d3[a, e]
+    try:
+        [lt] = oq.loss_types
+    except ValueError:
+        lt = oq.total_losses
+    return _dframe(dddict, csqidx, [lt]), dmgcsq
 
 
-def _dframe(adic, csqidx, loss_types):
-    # convert {eid, kid: dd} into a DataFrame (agg_id, event_id, loss_id)
+def _dframe(dddic, csqidx, loss_types):
+    # convert {(eid, kid): dd} into a DataFrame (agg_id, event_id, loss_id)
     dic = general.AccumDict(accum=[])
-    for (eid, kid), dd in sorted(adic.items()):
+    for (eid, kid), dd in sorted(dddic.items()):
         for li, lt in enumerate(loss_types):
             dic['agg_id'].append(kid)
             dic['event_id'].append(eid)
@@ -301,12 +310,12 @@ def post_execute(self, dummy):
         D = len(self.crmodel.damage_states)
         # fix no_damage distribution for events with zero damage
         number = self.assetcol['value-number']
-        nl = len(oq.loss_types)
+        L = len(oq.loss_types)
         for r in range(self.R):
             ne = prc.num_events[r]
             self.dmgcsq[:, r, 0] = (  # no damage
-                nl * number * ne - self.dmgcsq[:, r, 1:D].sum(axis=1))
-            self.dmgcsq[:, r] /= ne
+                number * ne * L - self.dmgcsq[:, r, 1:D].sum(axis=1))
+            self.dmgcsq[:, r] /= (ne * L)
         assert (self.dmgcsq >= 0).all()  # sanity check
         self.datastore['damages-rlzs'] = self.dmgcsq
         set_rlzs_stats(self.datastore,

openquake/calculators/multi_risk.py

@@ -59,11 +59,12 @@ def get_dmg_csq(crm, assets_by_site, gmf, time_event):
                         for k, w in zip(df.risk_id, df.weight)]
                 # NB: risk logic trees are not yet supported in multi_risk
                 fracs[li] = rm.scenario_damage(loss_type, assets, peril_df, 'peril')
-            csq = crm.compute_csq(assets, fracs, df, crm.loss_types, time_event)
+            csq = crm.compute_csq(
+                assets, fracs, df, crm.oqparam.total_loss_types, time_event)
             number = assets['value-number']
             for a, o in enumerate(assets['ordinal']):
                 out[o, :, 0, :D] = number[a] * fracs[:, a]
-                out[o, :, 0, [D]] = csq['losses'][:, a]
+                out[o, :, 0, [D]] = csq['losses'][a]
     return out
 
 

openquake/calculators/tests/scenario_damage_test.py

@@ -161,8 +161,8 @@ def test_case_7(self):
         df = self.calc.datastore.read_df(
             'risk_by_event', ['event_id', 'loss_id', 'agg_id'],
             dict(agg_id=K))
-        self.assertEqual(len(df), 300)
-        self.assertEqual(len(df[df.dmg_1 > 0]), 174)  # only 174/300 are nonzero
+        self.assertEqual(len(df), 100)
+        self.assertEqual(len(df[df.dmg_1 > 0]), 58)  # only 58/100 are nonzero
 
     def test_case_8(self):
         # case with a shakemap

openquake/calculators/views.py

@@ -552,6 +552,7 @@ def view_portfolio_loss(token, dstore):
     return text_table([['avg'] + avgs], ['loss'] + oq.loss_types)
 
 
+# used in the oq-risk-tests
 @view.add('portfolio_dmgdist')
 def portfolio_dmgdist(token, dstore):
     """
@@ -561,10 +562,10 @@ def portfolio_dmgdist(token, dstore):
     dstates = ['no_damage'] + oq.limit_states
     D = len(dstates)
     arr = dstore['damages-rlzs'][:, 0, :D].sum(axis=0)  # shape D
-    tbl = numpy.zeros(len(arr), dt(['total'] + dstates))
-    tbl['total'] = arr.sum(axis=1)
+    tbl = numpy.zeros(1, dt(['total'] + dstates))
+    tbl['total'] = arr.sum()
     for dsi, ds in enumerate(dstates):
-        tbl[ds] = arr[:, dsi]
+        tbl[ds] = arr[dsi]
     return tbl
 
 

openquake/commonlib/oqvalidation.py

@@ -1385,6 +1385,11 @@ def check_risk(self):
             if not self.investigation_time and self.hazard_calculation_id is None:
                 self.raise_invalid('missing investigation_time')
 
+        # check total_losses
+        if ('damage' in self.calculation_mode and len(self.loss_types) > 1
+                and not self.total_losses):
+            self.raise_invalid('you forgot to specify total_losses =')
+
     def check_ebrisk(self):
         # check specific to ebrisk
         if self.calculation_mode == 'ebrisk':
@@ -1748,14 +1753,15 @@ def ext_loss_types(self):
     @property
     def total_loss_types(self):
         """
-        :returns: the loss types in total_losses or the single loss type
+        :returns: a dictionary loss_type -> index
         """
         if self.total_losses:
-            return self.total_losses.split('+')
+            total = self.total_losses.split('+')
         elif len(self.loss_types) == 1:
-            return self.loss_types
+            total = self.loss_types
         else:
             self.raise_invalid('please specify total_losses')
+        return {lt: li for li, lt in enumerate(self.loss_types) if lt in total}
 
     def loss_dt(self, dtype=F64):
         """

openquake/qa_tests_data/scenario_damage/case_21/expected/aggrisk-parent.csv

@@ -1,20 +1,8 @@
-#,,,,,,,,,"generated_by='OpenQuake engine 3.22.0-git083e1be7c8', start_date='2024-10-22T16:53:41', checksum=2278520494, investigation_time=None, risk_investigation_time=None"
+#,,,,,,,,,"generated_by='OpenQuake engine 3.22.0-git2c7171503a', start_date='2024-10-23T08:10:47', checksum=2278520494, investigation_time=None, risk_investigation_time=None"
 loss_type,parent_id,rlz_id,no_damage,slight,moderate,extensive,complete,non_operational_value,non_operational_ratio
 structural,A,0,9.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,1.00000E-01,1.00000E-01,1.00000E-01
 structural,A,1,8.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,2.00000E-01,2.00000E-01,2.00000E-01
-liquefaction,A,0,9.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,1.00000E-01,1.00000E-01,1.00000E-01
-liquefaction,A,1,8.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,2.00000E-01,2.00000E-01,2.00000E-01
-landslide,A,0,9.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,1.00000E-01,1.00000E-01,1.00000E-01
-landslide,A,1,8.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,2.00000E-01,2.00000E-01,2.00000E-01
 structural,B,0,1.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00
 structural,B,1,1.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00
-liquefaction,B,0,1.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00
-liquefaction,B,1,1.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00
-landslide,B,0,1.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00
-landslide,B,1,1.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00
 structural,E1,0,-2.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,3.00000E+00,3.00000E+00,3.00000E+00
 structural,E1,1,-2.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,3.00000E+00,3.00000E+00,3.00000E+00
-liquefaction,E1,0,-2.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,3.00000E+00,3.00000E+00,3.00000E+00
-liquefaction,E1,1,-2.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,3.00000E+00,3.00000E+00,3.00000E+00
-landslide,E1,0,-2.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,3.00000E+00,3.00000E+00,3.00000E+00
-landslide,E1,1,-2.00000E+00,0.00000E+00,0.00000E+00,0.00000E+00,3.00000E+00,3.00000E+00,3.00000E+00

openquake/qa_tests_data/scenario_damage/case_21/expected/aggrisk.csv

@@ -1,8 +1,4 @@
-#,,,,,,,,"generated_by='OpenQuake engine 3.22.0-git083e1be7c8', start_date='2024-10-22T16:53:41', checksum=2278520494, investigation_time=None, risk_investigation_time=None"
+#,,,,,,,,"generated_by='OpenQuake engine 3.22.0-git2c7171503a', start_date='2024-10-23T08:10:47', checksum=2278520494, investigation_time=None, risk_investigation_time=None"
 loss_type,rlz_id,no_damage,slight,moderate,extensive,complete,non_operational_value,non_operational_ratio
 structural,0,-1.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,3.10000E+00,3.10000E+00,1.03333E+00
 structural,1,-2.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,3.20000E+00,3.20000E+00,1.06667E+00
-liquefaction,0,-1.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,3.10000E+00,3.10000E+00,1.03333E+00
-liquefaction,1,-2.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,3.20000E+00,3.20000E+00,1.06667E+00
-landslide,0,-1.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,3.10000E+00,3.10000E+00,1.03333E+00
-landslide,1,-2.00000E-01,0.00000E+00,0.00000E+00,0.00000E+00,3.20000E+00,3.20000E+00,1.06667E+00

openquake/qa_tests_data/scenario_damage/case_22/expected/aggrisk.csv

@@ -1,5 +1,3 @@
-#,,,,,,,"generated_by='OpenQuake engine 3.22.0-git083e1be7c8', start_date='2024-10-22T16:53:41', checksum=2138001544, investigation_time=None, risk_investigation_time=None"
+#,,,,,,,"generated_by='OpenQuake engine 3.22.0-git2c7171503a', start_date='2024-10-23T08:10:48', checksum=2138001544, investigation_time=None, risk_investigation_time=None"
 loss_type,no_damage,slight,moderate,extreme,complete,losses_value,losses_ratio
 structural,2.29684E+00,1.41803E+00,1.39786E+00,9.49061E-01,8.93820E+00,1.84217E+04,1.08299E-01
-liquefaction,2.29684E+00,1.41803E+00,1.39786E+00,9.49061E-01,8.93820E+00,1.84217E+04,INF
-landslide,2.29684E+00,1.41803E+00,1.39786E+00,9.49061E-01,8.93820E+00,1.84217E+04,INF