Skip to content

Era5

ERA5LReader

用于从minio中读取era5-land数据

Methods:

Name Description
open_dataset

从minio中读取era5-land数据
from_shp

通过已有的矢量数据范围从minio服务器读取era5-land数据
from_aoi

用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取era5-land数据
to_netcdf

读取数据并保存为本地nc文件
Source code in hydro_opendata/reader/minio.py 
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
class ERA5LReader:
    """
    用于从minio中读取era5-land数据

    Methods:
        open_dataset(data_variables, start_time, end_time, dataset, bbox): 从minio中读取era5-land数据
        from_shp(data_variables, start_time, end_time, dataset, shp): 通过已有的矢量数据范围从minio服务器读取era5-land数据
        from_aoi(data_variables, start_time, end_time, dataset, aoi): 用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取era5-land数据
        to_netcdf(data_variables, start_time, end_time, dataset, shp, resolution, save_file): 读取数据并保存为本地nc文件
    """

    def __init__(self):
        self._variables = [
            "10 metre U wind component",
            "10 metre V wind component",
            "2 metre dewpoint temperature",
            "2 metre temperature",
            "Evaporation",
            "Evaporation from bare soil",
            "Evaporation from open water surfaces excluding oceans",
            "Evaporation from the top of canopy",
            "Evaporation from vegetation transpiration",
            "Forecast albedo",
            "Lake bottom temperature",
            "Lake ice total depth",
            "Lake ice surface temperature",
            "Lake mix-layer depth",
            "Lake mix-layer temperature",
            "Lake shape factor",
            "Lake total layer temperature",
            "Leaf area index, high vegetation",
            "Leaf area index, low vegetation",
            "Potential evaporation",
            "Runoff",
            "Skin reservoir content",
            "Skin temperature",
            "Snow albedo",
            "Snow cover",
            "Snow density",
            "Snow depth",
            "Snow depth water equivalent",
            "Snow evaporation",
            "Snowfall",
            "Snowmelt",
            "Soil temperature level 1",
            "Soil temperature level 2",
            "Soil temperature level 3",
            "Soil temperature level 4",
            "Sub-surface runoff",
            "Surface latent heat flux",
            "Surface net solar radiation",
            "Surface net thermal radiation",
            "Surface pressure",
            "Surface runoff",
            "Surface sensible heat flux",
            "Surface solar radiation downwards",
            "Surface thermal radiation downwards",
            "Temperature of snow layer",
            "Total precipitation",
            "Volumetric soil water layer 1",
            "Volumetric soil water layer 2",
            "Volumetric soil water layer 3",
            "Volumetric soil water layer 4",
        ]

        self._accumulated = [
            "Evaporation",
            "Evaporation from bare soil",
            "Evaporation from open water surfaces excluding oceans",
            "Evaporation from the top of canopy",
            "Evaporation from vegetation transpiration",
            "Potential evaporation",
            "Runoff",
            "Snow evaporation",
            "Snowfall",
            "Snowmelt",
            "Sub-surface runoff",
            "Surface latent heat flux",
            "Surface net solar radiation",
            "Surface net thermal radiation",
            "Surface runoff",
            "Surface sensible heat flux",
            "Surface solar radiation downwards",
            "Surface thermal radiation downwards",
            "Total precipitation",
        ]

    def open_dataset(
        self,
        data_variables=["Total precipitation"],
        start_time=None,
        end_time=None,
        dataset="wis",
        bbox=None,
        time_chunks=24,
    ):
        """
        从minio服务器读取era5-land数据

        Args:
            data_variables (list): 数据变量列表
            start_time (datetime64): 开始时间
            end_time (datetime64): 结束时间
            dataset (str): wis或camels
            bbox (list|tuple): 四至范围
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """

        if end_time <= start_time:
            raise Exception("结束时间不能早于开始时间")

        if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
            raise Exception("四至范围格式错误")

        if dataset != "wis" and dataset != "camels":
            raise Exception("dataset参数错误")

        if dataset == "wis":
            self._dataset = "geodata"
        elif dataset == "camels":
            self._dataset = "camdata"

        with fs.open(
            os.path.join(bucket_name, f"{self._dataset}/era5_land/era5l.json")
        ) as f:
            cont = json.load(f)
            self._starttime = np.datetime64(cont["start"])
            self._endtime = np.datetime64(cont["end"])
            self._bbox = cont["bbox"]

        chunks = {"time": time_chunks}
        ds = xr.open_dataset(
            "reference://",
            engine="zarr",
            chunks=chunks,
            backend_kwargs={
                "consolidated": False,
                "storage_options": {
                    # no matter you run code in windows or linux, the bucket's format should be Linux style
                    # so we don't use os.join.path
                    "fo": f"s3://{bucket_name}/{self._dataset}/era5_land/era5_land_.json",
                    "target_protocol": "s3",
                    "target_options": ro,
                    "remote_protocol": "s3",
                    "remote_options": ro,
                },
            },
        )

        ds = ds.filter_by_attrs(long_name=lambda v: v in data_variables)
        ds = ds.rename({"longitude": "lon", "latitude": "lat"})
        ds = ds.transpose("time", "lon", "lat")

        start_time = max(start_time, self._starttime)
        end_time = min(end_time, self._endtime)
        times = slice(start_time, end_time)
        ds = ds.sel(time=times)

        bbox = regen_box(bbox, 0.1, 0)

        if bbox[0] < self._bbox[0]:
            left = self._bbox[0]
        else:
            left = bbox[0]

        if bbox[1] < self._bbox[1]:
            bottom = self._bbox[1]
        else:
            bottom = bbox[1]

        if bbox[2] > self._bbox[2]:
            right = self._bbox[2]
        else:
            right = bbox[2]

        if bbox[3] > self._bbox[3]:
            top = self._bbox[3]
        else:
            top = bbox[3]

        longitudes = slice(left - 0.00001, right + 0.00001)
        latitudes = slice(bottom - 0.00001, top + 0.00001)

        ds = ds.sortby("lat", ascending=True)
        ds = ds.sel(lon=longitudes, lat=latitudes)

        return ds

    def from_shp(
        self,
        data_variables=["Total precipitation"],
        start_time=None,
        end_time=None,
        dataset="wis",
        shp=None,
        time_chunks=24,
    ):
        """
        通过已有的矢量数据范围从minio服务器读取era5-land数据

        Args:
            data_variables (list): 数据变量列表
            start_time (datetime64): 开始时间
            end_time (datetime64): 结束时间
            dataset (str): wis或camels
            shp (str): 矢量数据路径
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """

        gdf = gpd.GeoDataFrame.from_file(shp)
        b = gdf.bounds
        bbox = regen_box(
            (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
            0.1,
            0,
        )

        ds = self.open_dataset(
            data_variables, start_time, end_time, dataset, bbox, time_chunks
        )

        return self.open_dataset(
            data_variables, start_time, end_time, bbox, time_chunks
        )

    def from_aoi(
        self,
        data_variables=["Total precipitation"],
        start_time=None,
        end_time=None,
        dataset="wis",
        aoi: gpd.GeoDataFrame = None,
        time_chunks=24,
    ):
        """
        用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取era5-land数据

        Args:
            data_variables (list): 数据变量列表
            start_time (datetime64): 开始时间
            end_time (datetime64): 结束时间
            dataset (str): wis或camels
            aoi (GeoDataFrame): 已有的GeoPandas.GeoDataFrame对象
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """

        b = aoi.bounds
        bbox = regen_box(
            (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
            0.1,
            0,
        )

        ds = self.open_dataset(
            data_variables, start_time, end_time, dataset, bbox, time_chunks
        )

        return ds

    def to_netcdf(
        self,
        data_variables=["Total precipitation"],
        start_time=None,
        end_time=None,
        dataset="wis",
        shp=None,
        resolution="hourly",
        save_file="era5.nc",
        time_chunks=24,
    ):
        """
        读取数据并保存为本地nc文件

        Args:
            data_variables (list): 数据变量列表
            start_time (datetime64): 开始时间
            end_time (datetime64): 结束时间
            dataset (str): wis或camels
            shp (str): 已有的矢量数据路径
            resolution (str): 输出的时间分辨率
            save_file (str): 输出的文件路径
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """

        gdf = gpd.GeoDataFrame.from_file(shp)
        b = gdf.bounds
        bbox = regen_box(
            (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
            0.1,
            0,
        )

        if resolution == "hourly":
            ds = self.open_dataset(
                data_variables, start_time, end_time, dataset, bbox, time_chunks
            )

            if ds.to_netcdf(save_file) == None:
                print(save_file, "已生成")
                ds = xr.open_dataset(save_file)
                return ds

        if resolution == "daily":
            start_time = np.datetime64(f"{str(start_time)[:10]}T01:00:00.000000000")
            end_time = np.datetime64(str(end_time)[:10]) + 1
            end_time = np.datetime64(f"{str(end_time)}T00:00:00.000000000")

            ds = self.open_dataset(
                data_variables, start_time, end_time, dataset, bbox, time_chunks
            )

            days = ds["time"].size // 24

            data_vars = {}
            for k, v in ds.data_vars.items():
                data_vars[k] = v.attrs

            daily_arr = []

            for var, attr in data_vars.items():
                a = ds[var].to_numpy()

                if attr["long_name"] in self._accumulated:
                    xlist = [x for x in range(a.shape[0]) if x % 24 != 23]
                    _a = np.delete(a, xlist, axis=0)

                    daily_arr.append(_a)

                else:
                    r = np.split(a, days, axis=0)
                    _r = [
                        np.expand_dims(np.mean(r[i], axis=0), axis=0)
                        for i in range(len(r))
                    ]
                    __r = np.concatenate(_r)

                    daily_arr.append(__r)

            lats = ds["lat"].to_numpy()
            lons = ds["lon"].to_numpy()

            start_time = np.datetime64(str(start_time)[:10])

            creatspinc(daily_arr, data_vars, lats, lons, start_time, save_file, "daily")

            new = xr.open_dataset(save_file)
            print(save_file, "已生成")
            return new

        if resolution == "6-hourly":
            start_time = np.datetime64(f"{str(start_time)[:10]}T01:00:00.000000000")
            end_time = np.datetime64(str(end_time)[:10]) + 1
            end_time = np.datetime64(f"{str(end_time)}T00:00:00.000000000")

            ds = self.open_dataset(
                data_variables, start_time, end_time, dataset, bbox, time_chunks
            )

            days = ds["time"].size // 6

            data_vars = {}
            for k, v in ds.data_vars.items():
                data_vars[k] = v.attrs

            daily_arr = []

            for var, attr in data_vars.items():
                a = ds[var].to_numpy()

                if attr["long_name"] in self._accumulated:
                    xlist = [x for x in range(a.shape[0]) if x % 6 != 5]
                    _a = np.delete(a, xlist, axis=0)

                    daily_arr.append(_a)

                else:
                    r = np.split(a, days, axis=0)
                    _r = [
                        np.expand_dims(np.mean(r[i], axis=0), axis=0)
                        for i in range(len(r))
                    ]
                    __r = np.concatenate(_r)

                    daily_arr.append(__r)

            lats = ds["lat"].to_numpy()
            lons = ds["lon"].to_numpy()

            # start_time = np.datetime64(f'{str(start_time)[:10]}')
            year = int(f"{str(start_time)[0:4]}")
            month = int(f"{str(start_time)[5:7]}")
            day = int(f"{str(start_time)[8:10]}")
            dt = datetime(year, month, day, 0, 0, 0)

            creatspinc(daily_arr, data_vars, lats, lons, dt, save_file, "6-hourly")

            new = xr.open_dataset(save_file)
            print(save_file, "已生成")
            return new

from_aoi(data_variables=['Total precipitation'], start_time=None, end_time=None, dataset='wis', aoi=None, time_chunks=24)

用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取era5-land数据

Parameters:

Name Type Description Default
data_variables list

数据变量列表

 ['Total precipitation']
start_time datetime64

开始时间

 None
end_time datetime64

结束时间

 None
dataset str

wis或camels

 'wis'
aoi GeoDataFrame

已有的GeoPandas.GeoDataFrame对象

 None
time_chunks int

分块数量

 24

Returns:

Name Type Description
dataset Dataset

读取结果
Source code in hydro_opendata/reader/minio.py 
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
def from_aoi(
    self,
    data_variables=["Total precipitation"],
    start_time=None,
    end_time=None,
    dataset="wis",
    aoi: gpd.GeoDataFrame = None,
    time_chunks=24,
):
    """
    用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取era5-land数据

    Args:
        data_variables (list): 数据变量列表
        start_time (datetime64): 开始时间
        end_time (datetime64): 结束时间
        dataset (str): wis或camels
        aoi (GeoDataFrame): 已有的GeoPandas.GeoDataFrame对象
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """

    b = aoi.bounds
    bbox = regen_box(
        (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
        0.1,
        0,
    )

    ds = self.open_dataset(
        data_variables, start_time, end_time, dataset, bbox, time_chunks
    )

    return ds

from_shp(data_variables=['Total precipitation'], start_time=None, end_time=None, dataset='wis', shp=None, time_chunks=24)

通过已有的矢量数据范围从minio服务器读取era5-land数据

Parameters:

Name Type Description Default
data_variables list

数据变量列表

 ['Total precipitation']
start_time datetime64

开始时间

 None
end_time datetime64

结束时间

 None
dataset str

wis或camels

 'wis'
shp str

矢量数据路径

 None
time_chunks int

分块数量

 24

Returns:

Name Type Description
dataset Dataset

读取结果
Source code in hydro_opendata/reader/minio.py 
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
def from_shp(
    self,
    data_variables=["Total precipitation"],
    start_time=None,
    end_time=None,
    dataset="wis",
    shp=None,
    time_chunks=24,
):
    """
    通过已有的矢量数据范围从minio服务器读取era5-land数据

    Args:
        data_variables (list): 数据变量列表
        start_time (datetime64): 开始时间
        end_time (datetime64): 结束时间
        dataset (str): wis或camels
        shp (str): 矢量数据路径
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """

    gdf = gpd.GeoDataFrame.from_file(shp)
    b = gdf.bounds
    bbox = regen_box(
        (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
        0.1,
        0,
    )

    ds = self.open_dataset(
        data_variables, start_time, end_time, dataset, bbox, time_chunks
    )

    return self.open_dataset(
        data_variables, start_time, end_time, bbox, time_chunks
    )

open_dataset(data_variables=['Total precipitation'], start_time=None, end_time=None, dataset='wis', bbox=None, time_chunks=24)

从minio服务器读取era5-land数据

Parameters:

Name Type Description Default
data_variables list

数据变量列表

 ['Total precipitation']
start_time datetime64

开始时间

 None
end_time datetime64

结束时间

 None
dataset str

wis或camels

 'wis'
bbox list | tuple

四至范围

 None
time_chunks int

分块数量

 24

Returns:

Name Type Description
dataset Dataset

读取结果
Source code in hydro_opendata/reader/minio.py 
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
def open_dataset(
    self,
    data_variables=["Total precipitation"],
    start_time=None,
    end_time=None,
    dataset="wis",
    bbox=None,
    time_chunks=24,
):
    """
    从minio服务器读取era5-land数据

    Args:
        data_variables (list): 数据变量列表
        start_time (datetime64): 开始时间
        end_time (datetime64): 结束时间
        dataset (str): wis或camels
        bbox (list|tuple): 四至范围
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """

    if end_time <= start_time:
        raise Exception("结束时间不能早于开始时间")

    if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
        raise Exception("四至范围格式错误")

    if dataset != "wis" and dataset != "camels":
        raise Exception("dataset参数错误")

    if dataset == "wis":
        self._dataset = "geodata"
    elif dataset == "camels":
        self._dataset = "camdata"

    with fs.open(
        os.path.join(bucket_name, f"{self._dataset}/era5_land/era5l.json")
    ) as f:
        cont = json.load(f)
        self._starttime = np.datetime64(cont["start"])
        self._endtime = np.datetime64(cont["end"])
        self._bbox = cont["bbox"]

    chunks = {"time": time_chunks}
    ds = xr.open_dataset(
        "reference://",
        engine="zarr",
        chunks=chunks,
        backend_kwargs={
            "consolidated": False,
            "storage_options": {
                # no matter you run code in windows or linux, the bucket's format should be Linux style
                # so we don't use os.join.path
                "fo": f"s3://{bucket_name}/{self._dataset}/era5_land/era5_land_.json",
                "target_protocol": "s3",
                "target_options": ro,
                "remote_protocol": "s3",
                "remote_options": ro,
            },
        },
    )

    ds = ds.filter_by_attrs(long_name=lambda v: v in data_variables)
    ds = ds.rename({"longitude": "lon", "latitude": "lat"})
    ds = ds.transpose("time", "lon", "lat")

    start_time = max(start_time, self._starttime)
    end_time = min(end_time, self._endtime)
    times = slice(start_time, end_time)
    ds = ds.sel(time=times)

    bbox = regen_box(bbox, 0.1, 0)

    if bbox[0] < self._bbox[0]:
        left = self._bbox[0]
    else:
        left = bbox[0]

    if bbox[1] < self._bbox[1]:
        bottom = self._bbox[1]
    else:
        bottom = bbox[1]

    if bbox[2] > self._bbox[2]:
        right = self._bbox[2]
    else:
        right = bbox[2]

    if bbox[3] > self._bbox[3]:
        top = self._bbox[3]
    else:
        top = bbox[3]

    longitudes = slice(left - 0.00001, right + 0.00001)
    latitudes = slice(bottom - 0.00001, top + 0.00001)

    ds = ds.sortby("lat", ascending=True)
    ds = ds.sel(lon=longitudes, lat=latitudes)

    return ds

to_netcdf(data_variables=['Total precipitation'], start_time=None, end_time=None, dataset='wis', shp=None, resolution='hourly', save_file='era5.nc', time_chunks=24)

读取数据并保存为本地nc文件

Parameters:

Name Type Description Default
data_variables list

数据变量列表

 ['Total precipitation']
start_time datetime64

开始时间

 None
end_time datetime64

结束时间

 None
dataset str

wis或camels

 'wis'
shp str

已有的矢量数据路径

 None
resolution str

输出的时间分辨率

 'hourly'
save_file str

输出的文件路径

 'era5.nc'
time_chunks int

分块数量

 24

Returns:

Name Type Description
dataset Dataset

读取结果
Source code in hydro_opendata/reader/minio.py 
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
def to_netcdf(
    self,
    data_variables=["Total precipitation"],
    start_time=None,
    end_time=None,
    dataset="wis",
    shp=None,
    resolution="hourly",
    save_file="era5.nc",
    time_chunks=24,
):
    """
    读取数据并保存为本地nc文件

    Args:
        data_variables (list): 数据变量列表
        start_time (datetime64): 开始时间
        end_time (datetime64): 结束时间
        dataset (str): wis或camels
        shp (str): 已有的矢量数据路径
        resolution (str): 输出的时间分辨率
        save_file (str): 输出的文件路径
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """

    gdf = gpd.GeoDataFrame.from_file(shp)
    b = gdf.bounds
    bbox = regen_box(
        (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
        0.1,
        0,
    )

    if resolution == "hourly":
        ds = self.open_dataset(
            data_variables, start_time, end_time, dataset, bbox, time_chunks
        )

        if ds.to_netcdf(save_file) == None:
            print(save_file, "已生成")
            ds = xr.open_dataset(save_file)
            return ds

    if resolution == "daily":
        start_time = np.datetime64(f"{str(start_time)[:10]}T01:00:00.000000000")
        end_time = np.datetime64(str(end_time)[:10]) + 1
        end_time = np.datetime64(f"{str(end_time)}T00:00:00.000000000")

        ds = self.open_dataset(
            data_variables, start_time, end_time, dataset, bbox, time_chunks
        )

        days = ds["time"].size // 24

        data_vars = {}
        for k, v in ds.data_vars.items():
            data_vars[k] = v.attrs

        daily_arr = []

        for var, attr in data_vars.items():
            a = ds[var].to_numpy()

            if attr["long_name"] in self._accumulated:
                xlist = [x for x in range(a.shape[0]) if x % 24 != 23]
                _a = np.delete(a, xlist, axis=0)

                daily_arr.append(_a)

            else:
                r = np.split(a, days, axis=0)
                _r = [
                    np.expand_dims(np.mean(r[i], axis=0), axis=0)
                    for i in range(len(r))
                ]
                __r = np.concatenate(_r)

                daily_arr.append(__r)

        lats = ds["lat"].to_numpy()
        lons = ds["lon"].to_numpy()

        start_time = np.datetime64(str(start_time)[:10])

        creatspinc(daily_arr, data_vars, lats, lons, start_time, save_file, "daily")

        new = xr.open_dataset(save_file)
        print(save_file, "已生成")
        return new

    if resolution == "6-hourly":
        start_time = np.datetime64(f"{str(start_time)[:10]}T01:00:00.000000000")
        end_time = np.datetime64(str(end_time)[:10]) + 1
        end_time = np.datetime64(f"{str(end_time)}T00:00:00.000000000")

        ds = self.open_dataset(
            data_variables, start_time, end_time, dataset, bbox, time_chunks
        )

        days = ds["time"].size // 6

        data_vars = {}
        for k, v in ds.data_vars.items():
            data_vars[k] = v.attrs

        daily_arr = []

        for var, attr in data_vars.items():
            a = ds[var].to_numpy()

            if attr["long_name"] in self._accumulated:
                xlist = [x for x in range(a.shape[0]) if x % 6 != 5]
                _a = np.delete(a, xlist, axis=0)

                daily_arr.append(_a)

            else:
                r = np.split(a, days, axis=0)
                _r = [
                    np.expand_dims(np.mean(r[i], axis=0), axis=0)
                    for i in range(len(r))
                ]
                __r = np.concatenate(_r)

                daily_arr.append(__r)

        lats = ds["lat"].to_numpy()
        lons = ds["lon"].to_numpy()

        # start_time = np.datetime64(f'{str(start_time)[:10]}')
        year = int(f"{str(start_time)[0:4]}")
        month = int(f"{str(start_time)[5:7]}")
        day = int(f"{str(start_time)[8:10]}")
        dt = datetime(year, month, day, 0, 0, 0)

        creatspinc(daily_arr, data_vars, lats, lons, dt, save_file, "6-hourly")

        new = xr.open_dataset(save_file)
        print(save_file, "已生成")
        return new

GFSReader

用于从minio中读取gpm数据

Attributes:

Name Type Description
variables dict

变量名称及缩写

Methods:

Name Description
open_dataset

从minio中读取gfs数据
from_shp

通过已有的矢量数据范围从minio服务器读取gfs数据
from_aoi

用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gfs数据
Source code in hydro_opendata/reader/minio.py 
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
class GFSReader:
    """
    用于从minio中读取gpm数据

    Attributes:
        variables (dict): 变量名称及缩写

    Methods:
        open_dataset(data_variables, creation_date, creation_time, bbox): 从minio中读取gfs数据
        from_shp(data_variables, creation_date, creation_time, shp): 通过已有的矢量数据范围从minio服务器读取gfs数据
        from_aoi(data_variables, creation_date, creation_time, aoi): 用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gfs数据
    """

    def __init__(self):
        self._variables = {
            "dswrf": "downward_shortwave_radiation_flux",
            "pwat": "precipitable_water_entire_atmosphere",
            "2r": "relative_humidity_2m_above_ground",
            "2sh": "specific_humidity_2m_above_ground",
            "2t": "temperature_2m_above_ground",
            "tcc": "total_cloud_cover_entire_atmosphere",
            "tp": "total_precipitation_surface",
            "10u": "u_component_of_wind_10m_above_ground",
            "10v": "v_component_of_wind_10m_above_ground",
        }

        self._default = "tp"

    @property
    def variables(self):
        return self._variables

    @property
    def default_variable(self):
        return self._default

    def set_default_variable(self, short_name):
        if short_name in self._variables.keys():
            self._default = short_name
        else:
            raise Exception("变量设置错误")

    def open_dataset(
        self,
        creation_date=np.datetime64("2022-09-01"),
        creation_time="00",
        dataset="wis",
        bbox=(115, 38, 136, 54),
        time_chunks=24,
    ):
        """
        从minio服务器读取gfs数据

        Args:
            creation_date (datetime64): 创建日期
            creation_time (datetime64): 创建时间，即00\06\12\18之一
            dataset (str): wis或camels
            bbox (list|tuple): 四至范围
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """

        if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
            raise Exception("四至范围格式错误")

        if dataset != "wis" and dataset != "camels":
            raise Exception("dataset参数错误")

        if dataset == "wis":
            self._dataset = "geodata"
        elif dataset == "camels":
            self._dataset = "camdata"

        with fs.open(os.path.join(bucket_name, f"{self._dataset}/gfs/gfs.json")) as f:
            cont = json.load(f)
            self._paras = cont

        short_name = self._default
        full_name = self._variables[short_name]

        start = np.datetime64(self._paras[short_name][0]["start"])
        end = np.datetime64(self._paras[short_name][-1]["end"])

        if creation_date < start or creation_date > end:
            print("超出时间范围！")
            return

        if creation_time not in ["00", "06", "12", "18"]:
            print("creation_time必须是00、06、12、18之一！")
            return

        year = str(creation_date.astype("object").year)
        month = str(creation_date.astype("object").month).zfill(2)
        day = str(creation_date.astype("object").day).zfill(2)

        change = np.datetime64("2022-09-01")
        if creation_date < change:
            json_url = f"s3://{bucket_name}/{self._dataset}/gfs/gfs_history/{year}/{month}/{day}/gfs{year}{month}{day}.t{creation_time}z.0p25.json"
        else:
            json_url = f"s3://{bucket_name}/{self._dataset}/gfs/{short_name}/{year}/{month}/{day}/gfs{year}{month}{day}.t{creation_time}z.0p25.json"

        chunks = {"valid_time": time_chunks}
        ds = xr.open_dataset(
            "reference://",
            engine="zarr",
            chunks=chunks,
            backend_kwargs={
                "consolidated": False,
                "storage_options": {
                    "fo": json_url,
                    "target_protocol": "s3",
                    "target_options": ro,
                    "remote_protocol": "s3",
                    "remote_options": ro,
                },
            },
        )

        if creation_date < change:
            ds = ds[full_name]
            box = self._paras[short_name][0]["bbox"]
        else:
            box = self.paras[short_name][-1]["bbox"]

        # ds = ds.filter_by_attrs(long_name=lambda v: v in data_variables)
        ds = ds.rename({"longitude": "lon", "latitude": "lat"})
        # ds = ds.transpose('time','valid_time','lon','lat')

        bbox = regen_box(bbox, 0.25, 0)

        if bbox[0] < box[0]:
            left = box[0]
        else:
            left = bbox[0]

        if bbox[1] < box[1]:
            bottom = box[1]
        else:
            bottom = bbox[1]

        if bbox[2] > box[2]:
            right = box[2]
        else:
            right = bbox[2]

        if bbox[3] > box[3]:
            top = box[3]
        else:
            top = bbox[3]

        longitudes = slice(left - 0.00001, right + 0.00001)
        latitudes = slice(bottom - 0.00001, top + 0.00001)

        ds = ds.sortby("lat", ascending=True)
        ds = ds.sel(lon=longitudes, lat=latitudes)

        return ds

    def from_shp(
        self,
        creation_date=np.datetime64("2022-09-01"),
        creation_time="00",
        dataset="wis",
        shp=None,
        time_chunks=24,
    ):
        """
        通过已有的矢量数据范围从minio服务器读取gfs数据

        Args:
            creation_date (datetime64): 创建日期
            creation_time (datetime64): 创建时间，即00\06\12\18之一
            dataset (str): wis或camels
            shp (str): 矢量数据路径
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """

        gdf = gpd.GeoDataFrame.from_file(shp)
        b = gdf.bounds
        bbox = regen_box(
            (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
            0.1,
            0,
        )

        ds = self.open_dataset(creation_date, creation_time, dataset, bbox, time_chunks)

        return ds

    def from_aoi(
        self,
        creation_date=np.datetime64("2022-09-01"),
        creation_time="00",
        dataset="wis",
        aoi: gpd.GeoDataFrame = None,
        time_chunks=24,
    ):
        """
        通过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gfs数据

        Args:
            creation_date (datetime64): 创建日期
            creation_time (datetime64): 创建时间，即00\06\12\18之一
            dataset (str): wis或camels
            aoi (GeoDataFrame): 已有的GeoPandas.GeoDataFrame对象
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """
        b = aoi.bounds
        bbox = regen_box(
            (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
            0.1,
            0,
        )

        ds = self.open_dataset(creation_date, creation_time, dataset, bbox, time_chunks)

        return ds

from_aoi(creation_date=np.datetime64('2022-09-01'), creation_time='00', dataset='wis', aoi=None, time_chunks=24) 

1
2
3
4
5
    通过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gfs数据

    Args:
        creation_date (datetime64): 创建日期
        creation_time (datetime64): 创建时间，即00

8之一 dataset (str): wis或camels aoi (GeoDataFrame): 已有的GeoPandas.GeoDataFrame对象 time_chunks (int): 分块数量

1
2
    Returns:
        dataset (Dataset): 读取结果
Source code in hydro_opendata/reader/minio.py 
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
def from_aoi(
    self,
    creation_date=np.datetime64("2022-09-01"),
    creation_time="00",
    dataset="wis",
    aoi: gpd.GeoDataFrame = None,
    time_chunks=24,
):
    """
    通过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gfs数据

    Args:
        creation_date (datetime64): 创建日期
        creation_time (datetime64): 创建时间，即00\06\12\18之一
        dataset (str): wis或camels
        aoi (GeoDataFrame): 已有的GeoPandas.GeoDataFrame对象
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """
    b = aoi.bounds
    bbox = regen_box(
        (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
        0.1,
        0,
    )

    ds = self.open_dataset(creation_date, creation_time, dataset, bbox, time_chunks)

    return ds

from_shp(creation_date=np.datetime64('2022-09-01'), creation_time='00', dataset='wis', shp=None, time_chunks=24) 

1
2
3
4
5
    通过已有的矢量数据范围从minio服务器读取gfs数据

    Args:
        creation_date (datetime64): 创建日期
        creation_time (datetime64): 创建时间，即00

8之一 dataset (str): wis或camels shp (str): 矢量数据路径 time_chunks (int): 分块数量

1
2
    Returns:
        dataset (Dataset): 读取结果
Source code in hydro_opendata/reader/minio.py 
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
def from_shp(
    self,
    creation_date=np.datetime64("2022-09-01"),
    creation_time="00",
    dataset="wis",
    shp=None,
    time_chunks=24,
):
    """
    通过已有的矢量数据范围从minio服务器读取gfs数据

    Args:
        creation_date (datetime64): 创建日期
        creation_time (datetime64): 创建时间，即00\06\12\18之一
        dataset (str): wis或camels
        shp (str): 矢量数据路径
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """

    gdf = gpd.GeoDataFrame.from_file(shp)
    b = gdf.bounds
    bbox = regen_box(
        (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
        0.1,
        0,
    )

    ds = self.open_dataset(creation_date, creation_time, dataset, bbox, time_chunks)

    return ds

open_dataset(creation_date=np.datetime64('2022-09-01'), creation_time='00', dataset='wis', bbox=(115, 38, 136, 54), time_chunks=24) 

1
2
3
4
5
    从minio服务器读取gfs数据

    Args:
        creation_date (datetime64): 创建日期
        creation_time (datetime64): 创建时间，即00

8之一 dataset (str): wis或camels bbox (list|tuple): 四至范围 time_chunks (int): 分块数量

1
2
    Returns:
        dataset (Dataset): 读取结果
Source code in hydro_opendata/reader/minio.py 
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
def open_dataset(
    self,
    creation_date=np.datetime64("2022-09-01"),
    creation_time="00",
    dataset="wis",
    bbox=(115, 38, 136, 54),
    time_chunks=24,
):
    """
    从minio服务器读取gfs数据

    Args:
        creation_date (datetime64): 创建日期
        creation_time (datetime64): 创建时间，即00\06\12\18之一
        dataset (str): wis或camels
        bbox (list|tuple): 四至范围
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """

    if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
        raise Exception("四至范围格式错误")

    if dataset != "wis" and dataset != "camels":
        raise Exception("dataset参数错误")

    if dataset == "wis":
        self._dataset = "geodata"
    elif dataset == "camels":
        self._dataset = "camdata"

    with fs.open(os.path.join(bucket_name, f"{self._dataset}/gfs/gfs.json")) as f:
        cont = json.load(f)
        self._paras = cont

    short_name = self._default
    full_name = self._variables[short_name]

    start = np.datetime64(self._paras[short_name][0]["start"])
    end = np.datetime64(self._paras[short_name][-1]["end"])

    if creation_date < start or creation_date > end:
        print("超出时间范围！")
        return

    if creation_time not in ["00", "06", "12", "18"]:
        print("creation_time必须是00、06、12、18之一！")
        return

    year = str(creation_date.astype("object").year)
    month = str(creation_date.astype("object").month).zfill(2)
    day = str(creation_date.astype("object").day).zfill(2)

    change = np.datetime64("2022-09-01")
    if creation_date < change:
        json_url = f"s3://{bucket_name}/{self._dataset}/gfs/gfs_history/{year}/{month}/{day}/gfs{year}{month}{day}.t{creation_time}z.0p25.json"
    else:
        json_url = f"s3://{bucket_name}/{self._dataset}/gfs/{short_name}/{year}/{month}/{day}/gfs{year}{month}{day}.t{creation_time}z.0p25.json"

    chunks = {"valid_time": time_chunks}
    ds = xr.open_dataset(
        "reference://",
        engine="zarr",
        chunks=chunks,
        backend_kwargs={
            "consolidated": False,
            "storage_options": {
                "fo": json_url,
                "target_protocol": "s3",
                "target_options": ro,
                "remote_protocol": "s3",
                "remote_options": ro,
            },
        },
    )

    if creation_date < change:
        ds = ds[full_name]
        box = self._paras[short_name][0]["bbox"]
    else:
        box = self.paras[short_name][-1]["bbox"]

    # ds = ds.filter_by_attrs(long_name=lambda v: v in data_variables)
    ds = ds.rename({"longitude": "lon", "latitude": "lat"})
    # ds = ds.transpose('time','valid_time','lon','lat')

    bbox = regen_box(bbox, 0.25, 0)

    if bbox[0] < box[0]:
        left = box[0]
    else:
        left = bbox[0]

    if bbox[1] < box[1]:
        bottom = box[1]
    else:
        bottom = bbox[1]

    if bbox[2] > box[2]:
        right = box[2]
    else:
        right = bbox[2]

    if bbox[3] > box[3]:
        top = box[3]
    else:
        top = bbox[3]

    longitudes = slice(left - 0.00001, right + 0.00001)
    latitudes = slice(bottom - 0.00001, top + 0.00001)

    ds = ds.sortby("lat", ascending=True)
    ds = ds.sel(lon=longitudes, lat=latitudes)

    return ds

GPMReader

用于从minio中读取gpm数据

Methods:

Name Description
open_dataset

从minio中读取gpm数据
from_shp

通过已有的矢量数据范围从minio服务器读取gpm数据
from_aoi

用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gpm数据
Source code in hydro_opendata/reader/minio.py 
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
class GPMReader:
    """
    用于从minio中读取gpm数据

    Methods:
        open_dataset(start_time, end_time, dataset, bbox, time_resolution): 从minio中读取gpm数据
        from_shp(start_time, end_time, dataset, shp, time_resolution): 通过已有的矢量数据范围从minio服务器读取gpm数据
        from_aoi(start_time, end_time, dataset, aoi, time_resolution): 用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gpm数据
    """

    def __init__(self):
        pass

    def _get_dataset(self, scale, start_time, end_time, bbox, time_chunks):
        year = str(start_time)[:4]
        month = str(start_time)[5:7].zfill(2)
        day = str(self._endtime)[8:10].zfill(2)

        if scale == "Y":
            minio_path = f"s3://{bucket_name}/{self._dataset}/gpm{self._time_resolution}/{year}/gpm{year}_inc.json"

        elif scale == "M":
            minio_path = f"s3://{bucket_name}/{self._dataset}/gpm{self._time_resolution}/{year}/{month}/gpm{year}{month}_inc.json"

        chunks = {"time": time_chunks}
        ds = xr.open_dataset(
            "reference://",
            engine="zarr",
            chunks=chunks,
            backend_kwargs={
                "consolidated": False,
                "storage_options": {
                    "fo": minio_path,
                    "target_protocol": "s3",
                    "target_options": ro,
                    "remote_protocol": "s3",
                    "remote_options": ro,
                },
            },
        )

        # if self._time_resolution == '1d':
        #     ds = cf2datetime(ds)

        ds = ds["precipitationCal"]
        # ds.to_dataframe().filter(['precipitationCal','precipitationUncal']).to_xarray()

        # ds = ds.rename({"longitude": "lon", "latitude": "lat"})
        ds = ds.transpose("time", "lon", "lat")

        times = slice(start_time, end_time)
        ds = ds.sel(time=times)

        left = bbox[0]
        right = bbox[2]
        bottom = bbox[1]
        top = bbox[3]

        longitudes = slice(left - 0.00001, right + 0.00001)
        latitudes = slice(bottom - 0.00001, top + 0.00001)

        ds = ds.sortby("lat", ascending=True)
        ds = ds.sel(lon=longitudes, lat=latitudes)

        return ds

    def open_dataset(
        self,
        start_time=np.datetime64("2023-01-01T00:00:00.000000000"),
        end_time=np.datetime64("2023-01-02T00:00:00.000000000"),
        dataset="wis",
        bbox=(121, 39, 122, 40),
        time_resolution="1d",
        time_chunks=48,
    ):
        """
        从minio服务器读取gpm数据

        Args:
            start_time (datetime64): 开始时间
            end_time (datetime64): 结束时间
            dataset (str): wis或camels
            bbox (list|tuple): 四至范围
            time_resolution (str): 1d或30m
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """

        if end_time <= start_time:
            raise Exception("结束时间不能早于开始时间")

        if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
            raise Exception("四至范围错误")

        if dataset != "wis" and dataset != "camels":
            raise Exception("dataset参数错误")

        if time_resolution != "1d" and time_resolution != "30m":
            raise Exception("time_resolution参数错误")

        # dataset_name = get_dataset_name()
        if dataset == "wis":
            self._dataset = "geodata"
        elif dataset == "camels":
            self._dataset = "camdata"

        if time_resolution == "1d":
            self._time_resolution = "1d"
        elif time_resolution == "30m":
            self._time_resolution = ""

        with fs.open(
            os.path.join(
                bucket_name,
                f"{self._dataset}/gpm{self._time_resolution}/gpm{self._time_resolution}.json",
            )
        ) as f:
            cont = json.load(f)
            self._starttime = np.datetime64(cont["start"])
            self._endtime = np.datetime64(cont["end"])
            self._bbox = cont["bbox"]

        if start_time < self._starttime:
            start_time = self._starttime

        if end_time > self._endtime:
            end_time = self._endtime

        bbox = regen_box(bbox, 0.1, 0.05)

        if bbox[0] < self._bbox[0]:
            bbox[0] = self._bbox[0]
        if bbox[1] < self._bbox[1]:
            bbox[1] = self._bbox[1]
        if bbox[2] > self._bbox[2]:
            bbox[2] = self._bbox[2]
        if bbox[3] > self._bbox[3]:
            bbox[3] = self._bbox[3]

        year_start = int(str(start_time)[:4])
        year_end = int(str(end_time)[:4])
        end_year = int(str(self._endtime)[:4])

        if year_end < end_year:
            if year_start == year_end:
                ds = self._get_dataset(
                    scale="Y",
                    start_time=start_time,
                    end_time=end_time,
                    bbox=bbox,
                    time_chunks=time_chunks,
                )
                return ds

            elif year_start < year_end:
                dss = []
                years = range(year_start, year_end + 1)
                for year in years:
                    if year == year_start:
                        dss.append(
                            self._get_dataset(
                                scale="Y",
                                start_time=start_time,
                                end_time=np.datetime64(
                                    f"{year}-12-31T23:30:00.000000000"
                                ),
                                bbox=bbox,
                                time_chunks=time_chunks,
                            )
                        )

                    elif year == year_end:
                        dss.append(
                            self._get_dataset(
                                scale="Y",
                                start_time=np.datetime64(
                                    f"{year}-01-01T00:00:00.000000000"
                                ),
                                end_time=end_time,
                                bbox=bbox,
                                time_chunks=time_chunks,
                            )
                        )

                    else:
                        dss.append(
                            self._get_dataset(
                                scale="Y",
                                start_time=np.datetime64(
                                    f"{year}-01-01T00:00:00.000000000"
                                ),
                                end_time=np.datetime64(
                                    f"{year}-12-31T23:30:00.000000000"
                                ),
                                bbox=bbox,
                                time_chunks=time_chunks,
                            )
                        )
                return xr.merge(dss)

        else:
            month_end = int(str(end_time)[5:7])
            end_month = int(str(self._endtime)[5:7])

            if year_start == year_end:
                month_start = int(str(start_time)[5:7])
                if month_start == month_end:
                    return self._get_dataset(
                        scale="M",
                        start_time=start_time,
                        end_time=end_time,
                        bbox=bbox,
                        time_chunks=time_chunks,
                    )
                dss = []
                for m in range(month_start, month_end + 1):
                    if m == month_start:
                        d = calendar.monthrange(year_start, m)[1]
                        dss.append(
                            self._get_dataset(
                                scale="M",
                                start_time=start_time,
                                end_time=np.datetime64(
                                    f"{year_start}-{str(m).zfill(2)}-{str(d).zfill(2)}T23:30:00.000000000"
                                ),
                                bbox=bbox,
                                time_chunks=time_chunks,
                            )
                        )
                    elif m == month_end:
                        dss.append(
                            self._get_dataset(
                                scale="M",
                                start_time=np.datetime64(
                                    f"{year_start}-{str(m).zfill(2)}-01T00:00:00.000000000"
                                ),
                                end_time=end_time,
                                bbox=bbox,
                                time_chunks=time_chunks,
                            )
                        )
                    else:
                        d = calendar.monthrange(year_start, m)[1]
                        dss.append(
                            self._get_dataset(
                                scale="M",
                                start_time=np.datetime64(
                                    f"{year_start}-{str(m).zfill(2)}-01T00:00:00.000000000"
                                ),
                                end_time=np.datetime64(
                                    f"{year_start}-{str(m).zfill(2)}-{str(d).zfill(2)}T23:30:00.000000000"
                                ),
                                bbox=bbox,
                                time_chunks=time_chunks,
                            )
                        )

            else:
                dss = []

                for y in range(year_start, year_end + 1):
                    if y == year_start:
                        dss.append(
                            self._get_dataset(
                                scale="Y",
                                start_time=start_time,
                                end_time=np.datetime64(f"{y}-12-31T23:30:00.000000000"),
                                bbox=bbox,
                                time_chunks=time_chunks,
                            )
                        )

                    elif y == year_end:
                        for m in range(1, month_end + 1):
                            if m == month_end:
                                dss.append(
                                    self._get_dataset(
                                        scale="M",
                                        start_time=np.datetime64(
                                            f"{str(y).zfill(4)}-{str(m).zfill(2)}-01T00:00:00.000000000"
                                        ),
                                        end_time=end_time,
                                        bbox=bbox,
                                        time_chunks=time_chunks,
                                    )
                                )
                            else:
                                d = calendar.monthrange(y, m)[1]
                                dss.append(
                                    self._get_dataset(
                                        scale="M",
                                        start_time=np.datetime64(
                                            f"{str(y).zfill(4)}-{str(m).zfill(2)}-01T00:00:00.000000000"
                                        ),
                                        end_time=np.datetime64(
                                            f"{str(y).zfill(4)}-{str(m).zfill(2)}-{str(d).zfill(2)}T23:30:00.000000000"
                                        ),
                                        bbox=bbox,
                                        time_chunks=time_chunks,
                                    )
                                )

                    else:
                        dss.append(
                            self._get_dataset(
                                scale="Y",
                                start_time=np.datetime64(
                                    f"{y}-01-01T00:00:00.000000000"
                                ),
                                end_time=np.datetime64(f"{y}-12-31T23:30:00.000000000"),
                                bbox=bbox,
                                time_chunks=time_chunks,
                            )
                        )

            return xr.merge(dss)

    def from_shp(
        self,
        start_time=np.datetime64("2023-01-01T00:00:00.000000000"),
        end_time=np.datetime64("2023-01-02T00:00:00.000000000"),
        dataset="wis",
        shp=None,
        time_resolution="1d",
        time_chunks=48,
    ):
        """
        通过已有的矢量数据范围从minio服务器读取gpm数据

        Args:
            start_time (datetime64): 开始时间
            end_time (datetime64): 结束时间
            dataset (str): wis或camels
            shp (str): 矢量数据路径
            time_resolution (str): 1d或30m
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """

        gdf = gpd.GeoDataFrame.from_file(shp)
        b = gdf.bounds
        bbox = regen_box(
            (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
            0.1,
            0.05,
        )

        ds = self.open_dataset(
            start_time, end_time, dataset, bbox, time_resolution, time_chunks
        )
        return ds

    def from_aoi(
        self,
        start_time=np.datetime64("2023-01-01T00:00:00.000000000"),
        end_time=np.datetime64("2023-01-02T00:00:00.000000000"),
        dataset="wis",
        aoi: gpd.GeoDataFrame = None,
        time_resolution="1d",
        time_chunks=48,
    ):
        """
        用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gpm数据

        Args:
            start_time (datetime64): 开始时间
            end_time (datetime64): 结束时间
            dataset (str): wis或camels
            aoi (GeoDataFrame): 已有的GeoPandas.GeoDataFrame对象
            time_resolution (str): 1d或30m
            time_chunks (int): 分块数量

        Returns:
            dataset (Dataset): 读取结果
        """

        b = aoi.bounds
        bbox = regen_box(
            (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
            0.1,
            0.05,
        )
        ds = self.open_dataset(
            start_time, end_time, dataset, bbox, time_resolution, time_chunks
        )
        return ds

from_aoi(start_time=np.datetime64('2023-01-01T00:00:00.000000000'), end_time=np.datetime64('2023-01-02T00:00:00.000000000'), dataset='wis', aoi=None, time_resolution='1d', time_chunks=48)

用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gpm数据

Parameters:

Name Type Description Default
start_time datetime64

开始时间

 datetime64('2023-01-01T00:00:00.000000000')
end_time datetime64

结束时间

 datetime64('2023-01-02T00:00:00.000000000')
dataset str

wis或camels

 'wis'
aoi GeoDataFrame

已有的GeoPandas.GeoDataFrame对象

 None
time_resolution str

1d或30m

 '1d'
time_chunks int

分块数量

 48

Returns:

Name Type Description
dataset Dataset

读取结果
Source code in hydro_opendata/reader/minio.py 
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
def from_aoi(
    self,
    start_time=np.datetime64("2023-01-01T00:00:00.000000000"),
    end_time=np.datetime64("2023-01-02T00:00:00.000000000"),
    dataset="wis",
    aoi: gpd.GeoDataFrame = None,
    time_resolution="1d",
    time_chunks=48,
):
    """
    用过已有的GeoPandas.GeoDataFrame对象从minio服务器读取gpm数据

    Args:
        start_time (datetime64): 开始时间
        end_time (datetime64): 结束时间
        dataset (str): wis或camels
        aoi (GeoDataFrame): 已有的GeoPandas.GeoDataFrame对象
        time_resolution (str): 1d或30m
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """

    b = aoi.bounds
    bbox = regen_box(
        (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
        0.1,
        0.05,
    )
    ds = self.open_dataset(
        start_time, end_time, dataset, bbox, time_resolution, time_chunks
    )
    return ds

from_shp(start_time=np.datetime64('2023-01-01T00:00:00.000000000'), end_time=np.datetime64('2023-01-02T00:00:00.000000000'), dataset='wis', shp=None, time_resolution='1d', time_chunks=48)

通过已有的矢量数据范围从minio服务器读取gpm数据

Parameters:

Name Type Description Default
start_time datetime64

开始时间

 datetime64('2023-01-01T00:00:00.000000000')
end_time datetime64

结束时间

 datetime64('2023-01-02T00:00:00.000000000')
dataset str

wis或camels

 'wis'
shp str

矢量数据路径

 None
time_resolution str

1d或30m

 '1d'
time_chunks int

分块数量

 48

Returns:

Name Type Description
dataset Dataset

读取结果
Source code in hydro_opendata/reader/minio.py 
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
def from_shp(
    self,
    start_time=np.datetime64("2023-01-01T00:00:00.000000000"),
    end_time=np.datetime64("2023-01-02T00:00:00.000000000"),
    dataset="wis",
    shp=None,
    time_resolution="1d",
    time_chunks=48,
):
    """
    通过已有的矢量数据范围从minio服务器读取gpm数据

    Args:
        start_time (datetime64): 开始时间
        end_time (datetime64): 结束时间
        dataset (str): wis或camels
        shp (str): 矢量数据路径
        time_resolution (str): 1d或30m
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """

    gdf = gpd.GeoDataFrame.from_file(shp)
    b = gdf.bounds
    bbox = regen_box(
        (b.loc[0]["minx"], b.loc[0]["miny"], b.loc[0]["maxx"], b.loc[0]["maxy"]),
        0.1,
        0.05,
    )

    ds = self.open_dataset(
        start_time, end_time, dataset, bbox, time_resolution, time_chunks
    )
    return ds

open_dataset(start_time=np.datetime64('2023-01-01T00:00:00.000000000'), end_time=np.datetime64('2023-01-02T00:00:00.000000000'), dataset='wis', bbox=(121, 39, 122, 40), time_resolution='1d', time_chunks=48)

从minio服务器读取gpm数据

Parameters:

Name Type Description Default
start_time datetime64

开始时间

 datetime64('2023-01-01T00:00:00.000000000')
end_time datetime64

结束时间

 datetime64('2023-01-02T00:00:00.000000000')
dataset str

wis或camels

 'wis'
bbox list | tuple

四至范围

 (121, 39, 122, 40)
time_resolution str

1d或30m

 '1d'
time_chunks int

分块数量

 48

Returns:

Name Type Description
dataset Dataset

读取结果
Source code in hydro_opendata/reader/minio.py 
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
def open_dataset(
    self,
    start_time=np.datetime64("2023-01-01T00:00:00.000000000"),
    end_time=np.datetime64("2023-01-02T00:00:00.000000000"),
    dataset="wis",
    bbox=(121, 39, 122, 40),
    time_resolution="1d",
    time_chunks=48,
):
    """
    从minio服务器读取gpm数据

    Args:
        start_time (datetime64): 开始时间
        end_time (datetime64): 结束时间
        dataset (str): wis或camels
        bbox (list|tuple): 四至范围
        time_resolution (str): 1d或30m
        time_chunks (int): 分块数量

    Returns:
        dataset (Dataset): 读取结果
    """

    if end_time <= start_time:
        raise Exception("结束时间不能早于开始时间")

    if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
        raise Exception("四至范围错误")

    if dataset != "wis" and dataset != "camels":
        raise Exception("dataset参数错误")

    if time_resolution != "1d" and time_resolution != "30m":
        raise Exception("time_resolution参数错误")

    # dataset_name = get_dataset_name()
    if dataset == "wis":
        self._dataset = "geodata"
    elif dataset == "camels":
        self._dataset = "camdata"

    if time_resolution == "1d":
        self._time_resolution = "1d"
    elif time_resolution == "30m":
        self._time_resolution = ""

    with fs.open(
        os.path.join(
            bucket_name,
            f"{self._dataset}/gpm{self._time_resolution}/gpm{self._time_resolution}.json",
        )
    ) as f:
        cont = json.load(f)
        self._starttime = np.datetime64(cont["start"])
        self._endtime = np.datetime64(cont["end"])
        self._bbox = cont["bbox"]

    if start_time < self._starttime:
        start_time = self._starttime

    if end_time > self._endtime:
        end_time = self._endtime

    bbox = regen_box(bbox, 0.1, 0.05)

    if bbox[0] < self._bbox[0]:
        bbox[0] = self._bbox[0]
    if bbox[1] < self._bbox[1]:
        bbox[1] = self._bbox[1]
    if bbox[2] > self._bbox[2]:
        bbox[2] = self._bbox[2]
    if bbox[3] > self._bbox[3]:
        bbox[3] = self._bbox[3]

    year_start = int(str(start_time)[:4])
    year_end = int(str(end_time)[:4])
    end_year = int(str(self._endtime)[:4])

    if year_end < end_year:
        if year_start == year_end:
            ds = self._get_dataset(
                scale="Y",
                start_time=start_time,
                end_time=end_time,
                bbox=bbox,
                time_chunks=time_chunks,
            )
            return ds

        elif year_start < year_end:
            dss = []
            years = range(year_start, year_end + 1)
            for year in years:
                if year == year_start:
                    dss.append(
                        self._get_dataset(
                            scale="Y",
                            start_time=start_time,
                            end_time=np.datetime64(
                                f"{year}-12-31T23:30:00.000000000"
                            ),
                            bbox=bbox,
                            time_chunks=time_chunks,
                        )
                    )

                elif year == year_end:
                    dss.append(
                        self._get_dataset(
                            scale="Y",
                            start_time=np.datetime64(
                                f"{year}-01-01T00:00:00.000000000"
                            ),
                            end_time=end_time,
                            bbox=bbox,
                            time_chunks=time_chunks,
                        )
                    )

                else:
                    dss.append(
                        self._get_dataset(
                            scale="Y",
                            start_time=np.datetime64(
                                f"{year}-01-01T00:00:00.000000000"
                            ),
                            end_time=np.datetime64(
                                f"{year}-12-31T23:30:00.000000000"
                            ),
                            bbox=bbox,
                            time_chunks=time_chunks,
                        )
                    )
            return xr.merge(dss)

    else:
        month_end = int(str(end_time)[5:7])
        end_month = int(str(self._endtime)[5:7])

        if year_start == year_end:
            month_start = int(str(start_time)[5:7])
            if month_start == month_end:
                return self._get_dataset(
                    scale="M",
                    start_time=start_time,
                    end_time=end_time,
                    bbox=bbox,
                    time_chunks=time_chunks,
                )
            dss = []
            for m in range(month_start, month_end + 1):
                if m == month_start:
                    d = calendar.monthrange(year_start, m)[1]
                    dss.append(
                        self._get_dataset(
                            scale="M",
                            start_time=start_time,
                            end_time=np.datetime64(
                                f"{year_start}-{str(m).zfill(2)}-{str(d).zfill(2)}T23:30:00.000000000"
                            ),
                            bbox=bbox,
                            time_chunks=time_chunks,
                        )
                    )
                elif m == month_end:
                    dss.append(
                        self._get_dataset(
                            scale="M",
                            start_time=np.datetime64(
                                f"{year_start}-{str(m).zfill(2)}-01T00:00:00.000000000"
                            ),
                            end_time=end_time,
                            bbox=bbox,
                            time_chunks=time_chunks,
                        )
                    )
                else:
                    d = calendar.monthrange(year_start, m)[1]
                    dss.append(
                        self._get_dataset(
                            scale="M",
                            start_time=np.datetime64(
                                f"{year_start}-{str(m).zfill(2)}-01T00:00:00.000000000"
                            ),
                            end_time=np.datetime64(
                                f"{year_start}-{str(m).zfill(2)}-{str(d).zfill(2)}T23:30:00.000000000"
                            ),
                            bbox=bbox,
                            time_chunks=time_chunks,
                        )
                    )

        else:
            dss = []

            for y in range(year_start, year_end + 1):
                if y == year_start:
                    dss.append(
                        self._get_dataset(
                            scale="Y",
                            start_time=start_time,
                            end_time=np.datetime64(f"{y}-12-31T23:30:00.000000000"),
                            bbox=bbox,
                            time_chunks=time_chunks,
                        )
                    )

                elif y == year_end:
                    for m in range(1, month_end + 1):
                        if m == month_end:
                            dss.append(
                                self._get_dataset(
                                    scale="M",
                                    start_time=np.datetime64(
                                        f"{str(y).zfill(4)}-{str(m).zfill(2)}-01T00:00:00.000000000"
                                    ),
                                    end_time=end_time,
                                    bbox=bbox,
                                    time_chunks=time_chunks,
                                )
                            )
                        else:
                            d = calendar.monthrange(y, m)[1]
                            dss.append(
                                self._get_dataset(
                                    scale="M",
                                    start_time=np.datetime64(
                                        f"{str(y).zfill(4)}-{str(m).zfill(2)}-01T00:00:00.000000000"
                                    ),
                                    end_time=np.datetime64(
                                        f"{str(y).zfill(4)}-{str(m).zfill(2)}-{str(d).zfill(2)}T23:30:00.000000000"
                                    ),
                                    bbox=bbox,
                                    time_chunks=time_chunks,
                                )
                            )

                else:
                    dss.append(
                        self._get_dataset(
                            scale="Y",
                            start_time=np.datetime64(
                                f"{y}-01-01T00:00:00.000000000"
                            ),
                            end_time=np.datetime64(f"{y}-12-31T23:30:00.000000000"),
                            bbox=bbox,
                            time_chunks=time_chunks,
                        )
                    )

        return xr.merge(dss)
Last update: 2023-11-02