# whisper-evaluate

Evaluate the effectiveness of the Whisper model.

## Install
从pypi安装:
```shell script
pip install whisper-evaluate
```
从本地安装:
```shell script
git clone https://git.xindong.com/fengyanglu/whisper-evaluate.git
cd whisper-evaluate
pip install dist/whisper_evaluate-1.2.0-py3-none-any.whl
```

## How to use

可参考 [examples](./examples) , [test.py](./test.py)

### 请求模型结果
`WhisperQuery`
```python
from whisper_evaluate import WhisperQuery

wq = WhisperQuery(url="http://localhost:8901/v1/models/whisper:predict",
                  source_file="test_data/test-data-standard-latest.csv",
                  target_file="test_result.csv",
                  semaphore=500)
wq.run()
```

`NetQuery`

网易的请求等后端的人给中转接口...

### 计算模型指标
`Report`
```python
from whisper_evaluate import Report
import pprint
# 有三个参数: 
# task_name: 任务名称
# xd_other_type: 语心除了整体和政治,需要计算的分类指标, 传入一个list 如: ["色情", "辱骂"]
# wy_other_type: 网易除了整体和政治,需要计算的分类指标
rep = Report("testv1.1")

# 一步计算语心及网易的评估指标
# TODO 等网易的数据拿到后再修改本接口
rep = Report("testv1.1")
result = rep.report("test_data/test-data-standard-latest.csv",
                    "test_data/test-data-standard-latest-whisper1.1-res.txt",
                    "test_data/test_antispam_0727.txt")
pprint.pprint(result)
rep.save()  # 保存检查文件及评估结果


 # 单独计算语心结果
result = rep.whisper_report(stand_file="test_data/test-data-standard-latest.csv",
                            xd_file="test_data/test-data-standard-latest-whisper1.1-res.txt")
pprint.pprint(result)


# 使用测试标签与结果合在一起的文件计算
result = rep.whisper_report(xd_file="test_data/standard-whisper-result.csv")
pprint.pprint(result)
rep.save()  # 保存检查文件及评估结果


# 单独计算网易结果
# 网易的计算方式暂时没更新 等拿到接口后再更新
result = rep.netease_report("test_data/test-data-standard-latest.csv",
                            "test_data/test_antispam_0727.txt")
pprint.pprint(result)
rep.save()  # 保存检查文件及评估结果

```


`WhisperStatistics`
```python
from whisper_evaluate import WhisperStatistics
import pprint

ws = WhisperStatistics(result_file="test_data/standard-whisper-result.csv")
xd_other_type = ["其他"]   # 其他需要计算的
ws.count_all(xd_other_type)
pprint.pprint(ws.indicator)
```

`NetEaseStatistics`
```python
from whisper_evaluate import NetEaseStatistics
import pprint

ns = NetEaseStatistics("test_data/test-data-standard-latest.csv", "test_data/test_antispam_0727.txt")

other_type = []
ns.count_all(other_type)

pprint.pprint(ns.indicator)
```


### 语境测试
`context_test_single`

`True` 表示通过,`False`表示未通过

```python
from whisper_evaluate import context_test_single

# 语境测试
label = ["给蛤蟆+1s", "蛤蟆", "[1,2]", "政治"]
predict = {'matched_words': [], 'review_words': [],
           'new_words': [{'word': '+1s', 'offset_start': 3, 'offset_end': 5, 'type': 'Politics'}]}
assert context_test_single(label, predict) is False
label = ["给蛤蟆+1s", "+1s", "[3,5]", "政治"]
predict = {'matched_words': [], 'review_words': [],
           'new_words': [{'word': '+1s', 'offset_start': 3, 'offset_end': 5, 'type': 'Politics'}]}
assert context_test_single(label, predict) is True
```

### 回归测试
`regression_test_single`

`True` 表示通过,`False`表示未通过

```python
from whisper_evaluate import regression_test_single
label = ["你们马后炮有一套的", "你们马", "[0,2]", "其他"]
predict = {'matched_words': [], 'review_words': [],
           'new_words': [{'word': '你们', 'offset_start': 0, 'offset_end': 1, 'type': 'Abuse'}]}
assert regression_test_single(label, predict) is False


label = ["习大大我爱你", "习大大", "[0,3]", "政治"]
predict = {'review_words': [], 'new_words': [],
           'matched_words': [{'word': '习大大', 'offset_start': 0, 'offset_end': 2, 'type': 'Politics'}]}
assert regression_test_single(label, predict) is True
```


参考[test.py](./test.py)
```shell script
from whisper_evaluate import Report
import pprint

if __name__ == '__main__':
    # 有三个参数: 
    # task_name: 任务名称
    # xd_other_type: 语心除了整体和政治,需要计算的分类指标, 传入一个list 如: ["色情", "辱骂"]
    # wy_other_type: 网易除了整体和政治,需要计算的分类指标
    rep = Report(task_name="testv1.1")
    # 生成评估报告
    # 三个参数是 标准测试数据文件路径/语心原始结果数据文件路径/网易原始结果数据文件路径 
    result = rep.report("test_data/test-data-standard-latest.csv",
                   "test_data/test-data-standard-latest-whisper1.1-res.txt",
                   "test_data/test_antispam_0727.txt")
    pprint.pprint(result)
    # 保存统计备查数据 和 计算结果
    rep.save()
    


    # 单独计算whisper结果
    result = rep.whisper_report("test_data/test-data-standard-latest.csv",
                   "test_data/test-data-standard-latest-whisper1.1-res.txt")
    pprint.pprint(result)
    
    # 单独计算netease结果
    result = rep.netease_report("test_data/test-data-standard-latest.csv",
                   "test_data/test_antispam_0727.txt")
    pprint.pprint(result)

    
```

