API Reference#

Here you can find high-level interfaces in the learnware package.

Market#

class learnware.market.LearnwareMarket(organizer: learnware.market.base.BaseOrganizer, searcher: learnware.market.base.BaseSearcher, checker_list: Optional[List[learnware.market.base.BaseChecker]] = None, **kwargs)#

Bases: object

Base interface for market, it provide the interface of search/add/detele/update learnwares

__init__(organizer: learnware.market.base.BaseOrganizer, searcher: learnware.market.base.BaseSearcher, checker_list: Optional[List[learnware.market.base.BaseChecker]] = None, **kwargs)#

add_learnware(zip_path: str, semantic_spec: dict, checker_names: Optional[List[str]] = None, **kwargs) → Tuple[str, int]#

Add a learnware into the market.

Parameters

zip_path (str) – Filepath for learnware model, a zipped file.
semantic_spec (dict) – semantic_spec for new learnware, in dictionary format.
checker_names (List[str], optional) – List contains checker names, by default None

Returns

str indicating model_id
int indicating the final learnware check_status

Return type

Tuple[str, int]

search_learnware(user_info: learnware.market.base.BaseUserInfo, check_status: Optional[int] = None, **kwargs) → learnware.market.base.SearchResults#

Search learnwares based on user_info from learnwares with check_status

Parameters

user_info (BaseUserInfo) – User information for searching learnwares
check_status (int, optional) –
- None: search from all learnwares
- Others: search from learnwares with check_status

Returns

Search results

Return type

SearchResults

update_learnware(id: str, zip_path: Optional[str] = None, semantic_spec: Optional[dict] = None, checker_names: Optional[List[str]] = None, check_status: Optional[int] = None, **kwargs) → int#

Update learnware with zip_path and semantic_specification

Parameters

id (str) – Learnware id
zip_path (str) – Filepath for learnware model, a zipped file.
semantic_spec (dict) – semantic_spec for new learnware, in dictionary format.
checker_names (List[str], optional) – List contains checker names, by default None.
check_status (int, optional) – A flag indicating whether the learnware is usable, by default None.

Returns

The final learnware check_status.

Return type

int

get_learnware_ids(top: Optional[int] = None, check_status: Optional[int] = None, **kwargs) → List[str]#

get the list of learnware ids

Parameters

top (int, optional) – The first top element to return, by default None
check_status (int, optional) –
- None: return all learnware ids
- Others: return learnware ids with check_status

Raises

List[str] – the first top ids

get_learnwares(top: Optional[int] = None, check_status: Optional[int] = None, **kwargs) → List[learnware.learnware.base.Learnware]#

get the list of learnwares

Parameters

top (int, optional) – The first top element to return, by default None
check_status (int, optional) –
- None: return all learnwares
- Others: return learnwares with check_status

Raises

List[Learnware] – the first top learnwares

class learnware.market.BaseUserInfo(id: Optional[str] = None, semantic_spec: Optional[dict] = None, stat_info: Optional[dict] = None)#

Bases: object

User Information for searching learnware

__init__(id: Optional[str] = None, semantic_spec: Optional[dict] = None, stat_info: Optional[dict] = None)#

Initializing user information

Parameters

id (str, optional) – user id, could be ignored in easy market
semantic_spec (dict, optional) – semantic_spec selected by user, by default dict()
stat_info (dict, optional) – statistical information uploaded by user, by default dict()

get_semantic_spec() → dict#

Return user semantic specifications

Returns: user semantic specifications
Return type: dict

update_stat_info(name: str, item: Any)#

Update stat_info by market

Parameters

name (str) – Name of stat_info
item (Any) – Statistical information calculated by market

Organizer#

class learnware.market.BaseOrganizer(market_id, **kwargs)#

Bases: object

__init__(market_id, **kwargs)#

reload_market(rebuild=False, **kwargs) → bool#

Reload the learnware organizer when server restared.

Returns: A flag indicating whether the market is reload successfully.
Return type: bool

add_learnware(zip_path: str, semantic_spec: dict, check_status: int) → Tuple[str, bool]#

Add a learnware into the market.

Note

Given a prediction of a certain time, all signals before this time will be prepared well.

Parameters

zip_path (str) – Filepath for learnware model, a zipped file.
semantic_spec (dict) – semantic_spec for new learnware, in dictionary format.

Returns

str indicating model_id
int indicating what the flag of learnware is added.

Return type

Tuple[str, int]

Raises

FileNotFoundError – file for model or statistical specification not found

delete_learnware(id: str) → bool#

Delete a learnware from market

Parameters: id (str) – id of learnware to be deleted
Returns: True if the target learnware is deleted successfully.
Return type: bool
Raises: Exception – Raise an excpetion when given id is NOT found in learnware list

update_learnware(id: str, zip_path: str, semantic_spec: dict, check_status: int) → bool#

Update Learnware with id and content to be updated.

Parameters: id (str) – id of target learnware.

get_learnware_by_ids(id: Union[str, List[str]]) → Union[learnware.learnware.base.Learnware, List[learnware.learnware.base.Learnware]]#

Get Learnware from market by id

Parameters

id (Union[str, List[str]]) – Given one id or a list of ids as target.

Returns

Return a Learnware object or a list of Learnware objects based on the type of input param.

The returned items are search results.
’None’ indicating the target id not found.

Return type

Union[Learnware, List[Learnware]]

get_learnware_zip_path_by_ids(ids: Union[str, List[str]]) → Union[learnware.learnware.base.Learnware, List[learnware.learnware.base.Learnware]]#

Get Zipped Learnware file by id

Parameters: ids (Union[str, List[str]]) – Give a id or a list of ids str: id of target learnware List[str]: A list of ids of target learnwares
Returns: Return the path for target learnware or list of path. None for Learnware NOT Found.
Return type: Union[Learnware, List[Learnware]]

get_learnware_dir_path_by_ids(ids: Union[str, List[str]]) → Union[learnware.learnware.base.Learnware, List[learnware.learnware.base.Learnware]]#

Get Learnware dir path by id

Parameters: ids (Union[str, List[str]]) – Give a id or a list of ids str: id of target learnware List[str]: A list of ids of target learnwares
Returns: Return the dir path for target learnware or list of path. None for Learnware NOT Found.
Return type: Union[Learnware, List[Learnware]]

get_learnware_ids(top: Optional[int] = None, check_status: Optional[int] = None) → List[str]#

get the list of learnware ids

Parameters

top (int, optional) – The first top element to return, by default None
check_status (int, optional) –
- None: return all learnware ids
- Others: return learnware ids with check_status

Raises

List[str] – the first top ids

get_learnwares(top: Optional[int] = None, check_status: Optional[int] = None) → List[learnware.learnware.base.Learnware]#

get the list of learnwares

Parameters

top (int, optional) – The first top element to return, by default None
check_status (int, optional) –
- None: return all learnwares
- Others: return learnwares with check_status

Raises

List[Learnware] – the first top learnwares

class learnware.market.EasyOrganizer(market_id, **kwargs)#

Bases: learnware.market.base.BaseOrganizer

reload_market(rebuild=False) → bool#

Reload the learnware organizer when server restarted.

Returns: A flag indicating whether the market is reloaded successfully.
Return type: bool

add_learnware(zip_path: str, semantic_spec: dict, check_status: int, learnware_id: Optional[str] = None) → Tuple[str, int]#

Add a learnware into the market.

Parameters

zip_path (str) – Filepath for learnware model, a zipped file.
semantic_spec (dict) – semantic_spec for new learnware, in dictionary format.
check_status (int) – A flag indicating whether the learnware is usable.
learnware_id (int) – A id in database for learnware

Returns

str indicating model_id
int indicating the final learnware check_status

Return type

Tuple[str, int]

delete_learnware(id: str) → bool#

Delete Learnware from market

Parameters: id (str) – Learnware to be deleted
Returns: True for successful operation. False for id not found.
Return type: bool

update_learnware(id: str, zip_path: Optional[str] = None, semantic_spec: Optional[dict] = None, check_status: Optional[int] = None)#

Update learnware with zip_path, semantic_specification and check_status

Parameters

id (str) – Learnware id
zip_path (str, optional) – Filepath for learnware model, a zipped file.
semantic_spec (dict, optional) – semantic_spec for new learnware, in dictionary format.
check_status (int, optional) – A flag indicating whether the learnware is usable.

Returns

The final learnware check_status.

Return type

int

get_learnware_by_ids(ids: Union[str, List[str]]) → Union[learnware.learnware.base.Learnware, List[learnware.learnware.base.Learnware]]#

Search learnware by id or list of ids.

Parameters: ids (Union[str, List[str]]) – Give a id or a list of ids str: id of target learnware List[str]: A list of ids of target learnwares
Returns: Return target learnware or list of target learnwares. None for Learnware NOT Found.
Return type: Union[Learnware, List[Learnware]]

get_learnware_zip_path_by_ids(ids: Union[str, List[str]]) → Union[learnware.learnware.base.Learnware, List[learnware.learnware.base.Learnware]]#

Get Zipped Learnware file by id

Parameters: ids (Union[str, List[str]]) – Give a id or a list of ids str: id of target learnware List[str]: A list of ids of target learnwares
Returns: Return the path for target learnware or list of path. None for Learnware NOT Found.
Return type: Union[Learnware, List[Learnware]]

get_learnware_dir_path_by_ids(ids: Union[str, List[str]]) → Union[learnware.learnware.base.Learnware, List[learnware.learnware.base.Learnware]]#

Get Learnware dir path by id

Parameters: ids (Union[str, List[str]]) – Give a id or a list of ids str: id of target learnware List[str]: A list of ids of target learnwares
Returns: Return the dir path for target learnware or list of path. None for Learnware NOT Found.
Return type: Union[Learnware, List[Learnware]]

get_learnware_ids(top: Optional[int] = None, check_status: Optional[int] = None) → List[str]#

Get learnware ids

Parameters

top (int, optional) – The first top learnware ids to return, by default None
check_status (bool, optional) –
- None: return all learnware ids
- Others: return learnware ids with check_status

Returns

Learnware ids

Return type

List[str]

get_learnwares(top: Optional[int] = None, check_status: Optional[int] = None) → List[learnware.learnware.base.Learnware]#

Get learnware list

Parameters

top (int, optional) – The first top learnwares to return, by default None
check_status (bool, optional) –
- None: return all learnwares
- Others: return learnwares with check_status

Returns

Learnware list

Return type

List[Learnware]

get_learnware_info_from_storage(learnware_id: str) → Dict#

return learnware zip path and semantic_specification from storage

Parameters

learnware_id (str) – learnware id

Returns

semantic_spec: semantic_specification
zip_path: zip_path
folder_path: folder_path
use_flag: use_flag

Return type

Dict

class learnware.market.HeteroMapTableOrganizer(market_id, **kwargs)#

Bases: learnware.market.easy.organizer.EasyOrganizer

reload_market(rebuild=False) → bool#

Reload the heterogeneous learnware organizer when server restarted.

Returns: A flag indicating whether the heterogeneous market is reloaded successfully.
Return type: bool

reset(market_id, rebuild=False, auto_update=False, auto_update_limit=100, **training_args)#

Reset the heterogeneous market with specified settings.

Parameters

market_id (str) – the heterogeneous market’s id
rebuild (bool, optional) – A flag indicating whether to reload market, by default False
auto_update (bool, optional) – A flag indicating whether to enable automatic updating of market mapping, by default False
auto_update_limit (int, optional) – The threshold for the number of learnwares required to trigger an automatic market mapping update, by default 100

add_learnware(zip_path: str, semantic_spec: dict, check_status: int, learnware_id: Optional[str] = None) → Tuple[str, int]#

Add a learnware into the heterogeneous learnware market.: Initiates an update of the market mapping if auto_update is True and the number of learnwares supporting training reaches auto_update_limit.

Parameters

zip_path (str) – Filepath for learnware model, a zipped file.
semantic_spec (dict) – semantic_spec for new learnware, in dictionary format.
check_status (int) – A flag indicating whether the learnware is usable.
learnware_id (str, optional) – A id in database for learnware

Returns

str indicating model_id
int indicating the final learnware check_status

Return type

Tuple[str, int]

delete_learnware(id: str) → bool#

Delete learnware from heterogeneous learnware market.: If a corresponding HeteroMapTableSpecification exists, it is also removed.

Parameters: id (str) – Learnware to be deleted
Returns: True for successful operation. False for id not found.
Return type: bool

update_learnware(id: str, zip_path: Optional[str] = None, semantic_spec: Optional[dict] = None, check_status: Optional[int] = None) → bool#

Update learnware with zip_path, semantic_specification and check_status.: If the learnware supports heterogeneous market training, its HeteroMapTableSpecification is also updated.

Parameters

id (str) – Learnware id
zip_path (str, optional) – Filepath for learnware model, a zipped file.
semantic_spec (dict, optional) – semantic_spec for new learnware, in dictionary format.
check_status (int, optional) – A flag indicating whether the learnware is usable.

Returns

The final learnware check_status.

Return type

int

reload_learnware(learnware_id: str)#

Reload learnware into heterogeneous learnware market.: If a corresponding HeteroMapTableSpecification exists, it is also reloaded.

Parameters: learnware_id (str) – Learnware to be reloaded

_update_learnware_hetero_spec(ids: Union[str, List[str]])#

Update learnware by ids, attempting to generate HeteroMapTableSpecification for them.

Parameters: ids (Union[str, List[str]]) – Give a id or a list of ids str: id of target learnware List[str]: A list of ids of target learnwares

_get_hetero_learnware_ids(ids: Union[str, List[str]]) → List[str]#

Get learnware ids that supports heterogeneous market training and search.

Parameters: ids (Union[str, List[str]]) – Give a id or a list of ids str: id of target learnware List[str]: A list of ids of target learnwares
Returns: Learnware ids
Return type: List[str]

generate_hetero_map_spec(user_info: learnware.market.base.BaseUserInfo) → learnware.specification.system.hetero_table.HeteroMapTableSpecification#

Generate HeteroMapTableSpecificaion based on user’s input description and statistical information.

Parameters: user_info (BaseUserInfo) – user_info contains semantic_spec and stat_info
Returns: The generated HeteroMapTableSpecification for user
Return type: HeteroMapTableSpecification

static train(learnware_list: List[learnware.learnware.base.Learnware], save_dir: str, **kwargs) → learnware.market.heterogeneous.organizer.hetero_map.HeteroMap#

Build the market mapping model using learnwares that supports heterogeneous market training.

Parameters

learnware_list (List[Learnware]) – The learnware list to train the market mapping
save_dir (str) – Filepath where the trained market mapping will be saved

Returns

The trained market mapping model

Return type

HeteroMap

Searcher#

class learnware.market.BaseSearcher(organizer: learnware.market.base.BaseOrganizer, **kwargs)#

Bases: object

__init__(organizer: learnware.market.base.BaseOrganizer, **kwargs)#

__call__(user_info: learnware.market.base.BaseUserInfo, check_status: Optional[int] = None) → learnware.market.base.SearchResults#

Search learnwares based on user_info from learnwares with check_status

Parameters

user_info (BaseUserInfo) – user_info contains semantic_spec and stat_info
check_status (int, optional) –
- None: search from all learnwares
- Others: search from learnwares with check_status

class learnware.market.EasySearcher(organizer: learnware.market.easy.organizer.EasyOrganizer)#

Bases: learnware.market.base.BaseSearcher

__init__(organizer: learnware.market.easy.organizer.EasyOrganizer)#

__call__(user_info: learnware.market.base.BaseUserInfo, check_status: Optional[int] = None, max_search_num: int = 5, search_method: str = 'greedy') → learnware.market.base.SearchResults#

Search learnwares based on user_info from learnwares with check_status

Parameters

user_info (BaseUserInfo) – user_info contains semantic_spec and stat_info
max_search_num (int) – The maximum number of the returned learnwares
check_status (int, optional) –
- None: search from all learnwares
- Others: search from learnwares with check_status

Returns

the first is the sorted list of rkme dist the second is the sorted list of Learnware (single) by the rkme dist the third is the score of Learnware (mixture) the fourth is the list of Learnware (mixture), the size is search_num

Return type

Tuple[List[float], List[Learnware], float, List[Learnware]]

class learnware.market.easy.EasyExactSemanticSearcher(organizer: learnware.market.base.BaseOrganizer, **kwargs)#

Bases: learnware.market.base.BaseSearcher

_match_semantic_spec(semantic_spec1, semantic_spec2)#: semantic_spec1: semantic spec input by user semantic_spec2: semantic spec in database

__call__(learnware_list: List[learnware.learnware.base.Learnware], user_info: learnware.market.base.BaseUserInfo) → learnware.market.base.SearchResults#

Search learnwares based on user_info from learnwares with check_status

Parameters

user_info (BaseUserInfo) – user_info contains semantic_spec and stat_info
check_status (int, optional) –
- None: search from all learnwares
- Others: search from learnwares with check_status

class learnware.market.easy.EasyFuzzSemanticSearcher(organizer: learnware.market.base.BaseOrganizer, **kwargs)#

Bases: learnware.market.base.BaseSearcher

_match_semantic_spec_tag(semantic_spec1, semantic_spec2) → bool#

Judge if tags of two semantic specs are consistent

Parameters

semantic_spec1 – semantic spec input by user
semantic_spec2 – semantic spec in database

Returns

consistent (True) or not consistent (False)

Return type

bool

__call__(learnware_list: List[learnware.learnware.base.Learnware], user_info: learnware.market.base.BaseUserInfo, max_num: int = 50000, min_score: float = 75.0) → learnware.market.base.SearchResults#

Search learnware by fuzzy matching of semantic spec

Parameters

learnware_list (List[Learnware]) – The list of learnwares
user_info (BaseUserInfo) – user_info contains semantic_spec
max_num (int, optional) – maximum number of learnwares returned, by default 50000
min_score (float, optional) – Minimum fuzzy matching score of learnwares returned, by default 30.0

Returns

The list of returned learnwares

Return type

List[Learnware]

class learnware.market.easy.EasyStatSearcher(organizer: learnware.market.base.BaseOrganizer, **kwargs)#

Bases: learnware.market.base.BaseSearcher

_convert_dist_to_score(dist_list: List[float], dist_ratio: float = 0.1, min_score: float = 0.92, improve_score: float = 0.7) → List[float]#

Convert mmd dist list into min_max score list

Parameters

dist_list (List[float]) – The list of mmd distances from learnware rkmes to user rkme
dist_ratio (float) – The paramter for converting mmd dist to score
min_score (float) – The minimum score for maximum returned score
improve_score (float) – The learnware score lower than improve_score will be improved

Returns

The list of min_max scores of each learnware

Return type

List[float]

_calculate_rkme_spec_mixture_weight(learnware_list: List[learnware.learnware.base.Learnware], user_rkme: learnware.specification.regular.table.rkme.RKMETableSpecification, intermediate_K: Optional[numpy.ndarray] = None, intermediate_C: Optional[numpy.ndarray] = None) → Tuple[List[float], float]#

Calculate mixture weight for the learnware_list based on a user’s rkme

Parameters

learnware_list (List[Learnware]) – A list of existing learnwares
user_rkme (RKMETableSpecification) – User RKME statistical specification
intermediate_K (np.ndarray, optional) – Intermediate kernel matrix K, by default None
intermediate_C (np.ndarray, optional) – Intermediate inner product vector C, by default None

Returns

The first is the list of mixture weights The second is the mmd dist between the mixture of learnware rkmes and the user’s rkme

Return type

Tuple[List[float], float]

_calculate_intermediate_K_and_C(learnware_list: List[learnware.learnware.base.Learnware], user_rkme: learnware.specification.regular.table.rkme.RKMETableSpecification, intermediate_K: Optional[numpy.ndarray] = None, intermediate_C: Optional[numpy.ndarray] = None) → Tuple[numpy.ndarray, numpy.ndarray]#

Incrementally update the values of intermediate_K and intermediate_C

Parameters

learnware_list (List[Learnware]) – The list of learnwares up till now
user_rkme (RKMETableSpecification) – User RKME statistical specification
intermediate_K (np.ndarray, optional) – Intermediate kernel matrix K, by default None
intermediate_C (np.ndarray, optional) – Intermediate inner product vector C, by default None

Returns

The first is the intermediate value of K The second is the intermediate value of C

Return type

Tuple[np.ndarray, np.ndarray]

_search_by_rkme_spec_mixture_auto(learnware_list: List[learnware.learnware.base.Learnware], user_rkme: learnware.specification.regular.table.rkme.RKMETableSpecification, max_search_num: int, weight_cutoff: float = 0.98) → Tuple[Optional[float], List[float], List[learnware.learnware.base.Learnware]]#

Select learnwares based on a total mixture ratio, then recalculate their mixture weights

Parameters

learnware_list (List[Learnware]) – The list of learnwares whose mixture approximates the user’s rkme
user_rkme (RKMETableSpecification) – User RKME statistical specification
max_search_num (int) – The maximum number of the returned learnwares
weight_cutoff (float, optional) – The ratio for selecting out the mose relevant learnwares, by default 0.9

Returns

The first is the mixture mmd dist The second is the list of weight The third is the list of Learnware

Return type

Tuple[float, List[float], List[Learnware]]

_filter_by_rkme_spec_single(sorted_score_list: List[float], learnware_list: List[learnware.learnware.base.Learnware], filter_score: float = 0.6, min_num: int = 1) → Tuple[List[float], List[learnware.learnware.base.Learnware]]#

Filter search result of _search_by_rkme_spec_single

Parameters

sorted_score_list (List[float]) – The list of score transformed by mmd dist
learnware_list (List[Learnware]) – The list of learnwares whose mixture approximates the user’s rkme
filter_score (float) – The learnware whose score is lower than filter_score will be filtered
min_num (int) – The minimum number of returned learnwares

Returns

the first is the list of score the second is the list of Learnware

Return type

Tuple[List[float], List[Learnware]]

_filter_by_rkme_spec_metadata(learnware_list: List[learnware.learnware.base.Learnware], user_rkme: Union[learnware.specification.regular.table.rkme.RKMETableSpecification, learnware.specification.regular.image.rkme.RKMEImageSpecification, learnware.specification.regular.text.rkme.RKMETextSpecification]) → List[learnware.learnware.base.Learnware]#

Filter learnwares whose rkme metadata different from user_rkme

Parameters

learnware_list (List[Learnware]) – The list of learnwares whose mixture approximates the user’s rkme
user_rkme (Union[RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification]) – User RKME statistical specification

Returns

Learnwares whose rkme dimensions equal user_rkme in user_info

Return type

List[Learnware]

_search_by_rkme_spec_mixture_greedy(learnware_list: List[learnware.learnware.base.Learnware], user_rkme: learnware.specification.regular.table.rkme.RKMETableSpecification, max_search_num: int, decay_rate: float = 0.95) → Tuple[Optional[float], List[float], List[learnware.learnware.base.Learnware]]#

Greedily match learnwares such that their mixture become closer and closer to user’s rkme

Parameters

learnware_list (List[Learnware]) – The list of learnwares whose mixture approximates the user’s rkme
user_rkme (RKMETableSpecification) – User RKME statistical specification
max_search_num (int) – The maximum number of the returned learnwares
decay_rate (float) – The decrease ratio of minimum mmd dist to stop further rkme_spec matching

Returns

The first is the mixture mmd dist The second is the list of weight The third is the list of Learnware

Return type

Tuple[float, List[float], List[Learnware]]

_search_by_rkme_spec_single(learnware_list: List[learnware.learnware.base.Learnware], user_rkme: Union[learnware.specification.regular.table.rkme.RKMETableSpecification, learnware.specification.regular.image.rkme.RKMEImageSpecification, learnware.specification.regular.text.rkme.RKMETextSpecification]) → Tuple[List[float], List[learnware.learnware.base.Learnware]]#

Calculate the distances between learnwares in the given learnware_list and user_rkme

Parameters

learnware_list (List[Learnware]) – The list of learnwares whose mixture approximates the user’s rkme
user_rkme (Union[RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification]) – user RKME statistical specification

Returns

the first is the list of mmd dist the second is the list of Learnware both lists are sorted by mmd dist

Return type

Tuple[List[float], List[Learnware]]

__call__(learnware_list: List[learnware.learnware.base.Learnware], user_info: learnware.market.base.BaseUserInfo, max_search_num: int = 5, search_method: str = 'greedy') → learnware.market.base.SearchResults#

Search learnwares based on user_info from learnwares with check_status

Parameters

user_info (BaseUserInfo) – user_info contains semantic_spec and stat_info
check_status (int, optional) –
- None: search from all learnwares
- Others: search from learnwares with check_status

class learnware.market.HeteroSearcher(organizer: learnware.market.easy.organizer.EasyOrganizer)#

Bases: learnware.market.easy.searcher.EasySearcher

Search learnwares based on user_info from learnwares with check_status.: Employs heterogeneous learnware search if specific requirements are met, otherwise resorts to homogeneous search methods.

Parameters

user_info (BaseUserInfo) – user_info contains semantic_spec and stat_info
max_search_num (int) – The maximum number of the returned learnwares
check_status (int, optional) –
- None: search from all learnwares
- Others: search from learnwares with check_status

Returns

Return type

Tuple[List[float], List[Learnware], float, List[Learnware]]

Checker#

class learnware.market.BaseChecker#

Bases: object

__call__(learnware: learnware.learnware.base.Learnware) → Tuple[int, str]#

Check the utility of a learnware

Parameters

learnware (Learnware) –

Returns

flag and message of learnware check result - int

A flag indicating whether the learnware can be accepted. - The INVALID_LEARNWARE denotes the learnware does not pass the check - The NOPREDICTION_LEARNWARE denotes the learnware pass the check but cannot make prediction due to some env dependency - The NOPREDICTION_LEARNWARE denotes the leanrware pass the check and can make prediction

str
A message indicating the reason of learnware check result

Return type

Tuple[int, str]

class learnware.market.EasySemanticChecker#

Bases: learnware.market.base.BaseChecker

__call__(learnware)#

Check the utility of a learnware

Parameters

learnware (Learnware) –

Returns

flag and message of learnware check result - int

A flag indicating whether the learnware can be accepted. - The INVALID_LEARNWARE denotes the learnware does not pass the check - The NOPREDICTION_LEARNWARE denotes the learnware pass the check but cannot make prediction due to some env dependency - The NOPREDICTION_LEARNWARE denotes the leanrware pass the check and can make prediction

str
A message indicating the reason of learnware check result

Return type

Tuple[int, str]

class learnware.market.EasyStatChecker#

Bases: learnware.market.base.BaseChecker

__call__(learnware)#

Check the utility of a learnware

Parameters

learnware (Learnware) –

Returns

flag and message of learnware check result - int

A flag indicating whether the learnware can be accepted. - The INVALID_LEARNWARE denotes the learnware does not pass the check - The NOPREDICTION_LEARNWARE denotes the learnware pass the check but cannot make prediction due to some env dependency - The NOPREDICTION_LEARNWARE denotes the leanrware pass the check and can make prediction

str
A message indicating the reason of learnware check result

Return type

Tuple[int, str]

Learnware#

class learnware.learnware.Learnware(id: str, model: Union[learnware.model.base.BaseModel, dict], specification: learnware.specification.base.Specification, learnware_dirpath: str)#

Bases: object

The learnware class, which is the basic components in learnware market

__init__(id: str, model: Union[learnware.model.base.BaseModel, dict], specification: learnware.specification.base.Specification, learnware_dirpath: str)#

The initialization method for learnware.

Parameters

id (str) – The learnware id that is generated by market, and is unique
model (Union[BaseModel, dict]) –
The learnware model for prediction, can be BaseModel or dict
- If the model is BaseModel, it denotes the model instant itself
- If the model is dict, it must be the following format:
  
  {
  “class_name”: str, “module_path”: str “kwargs”: dict,
  
  } - The class_name denotes the class name of model - The module_path denotes the module path of model - The kwards denotes the arguments of model, which is optional
specification (Specification) – The specification including the semantic specification and the statistic specification
dirpath (str) – The path of the learnware directory

Reuser#

class learnware.reuse.BaseReuser(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None)#

Bases: object

Providing the interfaces to reuse the learnwares which is searched by learnware

__init__(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None)#

The initializaiton method for base reuser

Parameters: learnware_list (List[Learnware]) – The learnware list to reuse and make predictions

predict(user_data: numpy.ndarray) → numpy.ndarray#

Give the final prediction for user data with reused learnware

Parameters: user_data (np.ndarray) – User’s unlabeled raw data.
Returns: The final prediction for user data with reused learnware
Return type: np.ndarray

Data Independent Reuser#

class learnware.reuse.JobSelectorReuser(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None, herding_num: int = 1000, use_herding: bool = True)#

Bases: learnware.reuse.base.BaseReuser

Baseline Multiple Learnware Reuser using Job Selector Method

__init__(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None, herding_num: int = 1000, use_herding: bool = True)#

The initialization method for job selector reuser

Parameters

learnware_list (List[Learnware]) – The learnware list, which should have RKME Specification for each learnweare
herding_num (int, optional) – The herding number, by default 1000

predict(user_data: Union[numpy.ndarray, List[str]]) → numpy.ndarray#

Give prediction for user data using baseline job-selector method

Parameters: user_data (Union[np.ndarray, List[str]]) – User’s unlabeled raw data.
Returns: Prediction given by job-selector method
Return type: np.ndarray

job_selector(user_data: numpy.ndarray)#

Train job selector based on user’s data, which predicts which learnware in the pool should be selected

Parameters: user_data (np.ndarray) – User’s raw data.

_calculate_rkme_spec_mixture_weight(user_data: numpy.ndarray, task_rkme_list: List[learnware.specification.regular.table.rkme.RKMETableSpecification], task_rkme_matrix: numpy.ndarray) → List[float]#

Calculate mixture weight for the learnware_list based on user’s data

Parameters

user_data (np.ndarray) – Raw user data.
task_rkme_list (List[RKMETableSpecification]) – The list of learnwares’ rkmes whose mixture approximates the user’s rkme
task_rkme_matrix (np.ndarray) – Inner product matrix calculated from task_rkme_list.

_selector_grid_search(org_train_x: numpy.ndarray, org_train_y: numpy.ndarray, train_x: numpy.ndarray, train_y: numpy.ndarray, val_x: numpy.ndarray, val_y: numpy.ndarray, num_class: int)#

Train a LGBMClassifier as job selector using the herding data as training instances.

Parameters

org_train_x (np.ndarray) – The original herding features.
org_train_y (np.ndarray) – The original hearding labels(which are learnware indexes).
train_x (np.ndarray) – Herding features used for training.
train_y (np.ndarray) – Herding labels used for training.
val_x (np.ndarray) – Herding features used for validation.
val_y (np.ndarray) – Herding labels used for validation.
num_class (int) – Total number of classes for the job selector(which is exactly the total number of learnwares to be reused).

Returns

The job selector model.

Return type

LGBMClassifier

class learnware.reuse.AveragingReuser(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None, mode: str = 'mean')#

Bases: learnware.reuse.base.BaseReuser

Baseline Multiple Learnware Reuser using Ensemble Method

__init__(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None, mode: str = 'mean')#

The initialization method for averaging ensemble reuser

Parameters

learnware_list (List[Learnware]) – The list contains learnwares.
mode (str) –
- “mean”: average the output of all learnwares for regression task (learnware output is a real number)
- ”vote_by_label”: vote by labels for classification task, learnware output belongs to the set {0, 1, …, class_num}
- ”vote_by_prob”: vote by probabilities for classification task, learnware output is a logits vector, denoting the probability of each class

predict(user_data: numpy.ndarray) → numpy.ndarray#

Prediction for user data using baseline ensemble method

Parameters: user_data (np.ndarray) – Raw user data.
Returns: Prediction given by ensemble method
Return type: np.ndarray

Data Dependent Reuser#

class learnware.reuse.EnsemblePruningReuser(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None, mode: str = 'classification')#

Bases: learnware.reuse.base.BaseReuser

Baseline Multiple Learnware Reuser uing Marign Distribution guided multi-objective evolutionary Ensemble Pruning (MDEP) Method.

References: [1] Yu-Chang Wu, Yi-Xiao He, Chao Qian, and Zhi-Hua Zhou. Multi-objective evolutionary ensemble pruning guided by margin distribution. In: Proceedings of the 17th International Conference on Parallel Problem Solving from Nature (PPSN’22), 2022, pp.427-441.

__init__(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None, mode: str = 'classification')#

The initialization method for ensemble pruning reuser

Parameters

learnware_list (List[Learnware]) – The list contains learnwares
mode (str) –
- “regression” for regression task (learnware output is a real number)
- ”classification” for classification task (learnware output is a logitis vector or belongs to the set {0, 1, …, class_num})

_MEDP_regression(v_predict: numpy.ndarray, v_true: numpy.ndarray, maxgen: int)#

Selective ensemble for regression model

Parameters

v_predict (np.ndarray) –
- The output of models on validation set.
- The dimension is (number of instances, number of models).
v_true (np.ndarray) –
- The ground truth of validation set.
- The dimension is (number of instances, 1).
maxgen (int) –
- The maximum number of iteration rounds.

Returns

Binary one-dimensional vector, 1 indicates that the corresponding model is selected.

Return type

np.ndarray

_MEDP_multiclass(v_predict: numpy.ndarray, v_true: numpy.ndarray, maxgen: int)#

Selective ensemble for multi-classification model

Parameters

v_predict (np.ndarray) –
- The output of models on validation set.
- The dimension is (number of instances, number of models).
v_true (np.ndarray) –
- The ground truth of validation set.
- The dimension is (number of instances, 1).
maxgen (int) –
- The maximum number of iteration rounds.

Returns

Binary one-dimensional vector, 1 indicates that the corresponding model is selected.

Return type

np.ndarray

_MEDP_binaryclass(v_predict: numpy.ndarray, v_true: numpy.ndarray, maxgen: int)#

Selective ensemble for binary classification model

Parameters

v_predict (np.ndarray) –
- The output of models on validation set.
- The dimension is (number of instances, number of models).
v_true (np.ndarray) –
- The ground truth of validation set.
- The dimension is (number of instances, 1).
maxgen (int) –
- The maximum number of iteration rounds.

Returns

Binary one-dimensional vector, 1 indicates that the corresponding model is selected.

Return type

np.ndarray

_get_predict(X: numpy.ndarray, selected_idxes: List[int])#

Concatenate the output of learnwares corresponding to selected_idxes

Parameters

X (np.ndarray) – Data that needs to be predicted
selected_idxes (List[int]) – Learnware index list

Returns

Prediction given by each selected learnware

Return type

np.ndarray

fit(val_X: numpy.ndarray, val_y: numpy.ndarray, maxgen: int = 500)#

Ensemble pruning based on the validation set

Parameters

val_X (np.ndarray) – Features of validation data.
val_y (np.ndarray) – Labels of validation data.
maxgen (int) – The maximum number of iteration rounds in ensemble pruning algorithms.

predict(user_data: numpy.ndarray) → numpy.ndarray#

Prediction for user data using the final pruned ensemble

Parameters: user_data (np.ndarray) – Raw user data.
Returns: Prediction given by ensemble method
Return type: np.ndarray

class learnware.reuse.FeatureAugmentReuser(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None, mode: Optional[str] = None)#

Bases: learnware.reuse.base.BaseReuser

FeatureAugmentReuser is a class for augmenting features using predictions of a given learnware model and applying regression or classification on the augmented dataset.

This class supports two modes:

“regression”: Uses RidgeCV for regression tasks.
“classification”: Uses LogisticRegressionCV for classification tasks.

__init__(learnware_list: Optional[List[learnware.learnware.base.Learnware]] = None, mode: Optional[str] = None)#

Initialize the FeatureAugmentReuser with a learnware model and a mode.

Parameters

learnware (List[Learnware]) – The list contains learnwares.
mode (str) – The mode of operation, either “regression” or “classification”.

predict(user_data: numpy.ndarray) → numpy.ndarray#

Predict the output for user data using the trained output aligner model.

Parameters: user_data (np.ndarray) – Input data for making predictions.
Returns: Predicted output from the output aligner model.
Return type: np.ndarray

fit(x_train: numpy.ndarray, y_train: numpy.ndarray)#

Train the output aligner model using the training data augmented with predictions from the learnware model.

Parameters

x_train (np.ndarray) – Training data features.
y_train (np.ndarray) – Training data labels.

_get_augment_data(X: numpy.ndarray) → numpy.ndarray#

Get the augmented data with model output.

Parameters: X (np.ndarray) – Input data.
Returns: Augment data with model output.
Return type: np.ndarray
Raises: TypeError – If the type of model output not in [np.ndarray, torch.Tensor].

Aligned Learnware#

class learnware.reuse.AlignLearnware(learnware: learnware.learnware.base.Learnware)#

Bases: learnware.learnware.base.Learnware

The aligned learnware class, providing the interfaces to align learnware and make predictions

__init__(learnware: learnware.learnware.base.Learnware)#

The initialization method for align learnware

Parameters: learnware (Learnware) – The learnware list to reuse and make predictions

align()#: Align the learnware with specification or data

class learnware.reuse.FeatureAlignLearnware(learnware: learnware.learnware.base.Learnware, cuda_idx=None, **align_arguments)#

Bases: learnware.reuse.align.AlignLearnware

FeatureAlignLearnware is a class for aligning features from a user dataset with a target dataset using a learnware model. It supports both classification and regression tasks and uses a feature alignment trainer for alignment.

learnware#

The learnware model used for final prediction.

Type: Learnware

align_arguments#

Additional arguments for the feature alignment trainer.

Type: dict

cuda_idx#

Index of the CUDA device to be used for computations.

Type: int

__init__(learnware: learnware.learnware.base.Learnware, cuda_idx=None, **align_arguments)#

Initialize the FeatureAlignLearnware with a learnware model, mode, CUDA device index, and alignment arguments.

Parameters

learnware (Learnware) – A learnware model used for initial predictions.
cuda_idx (int) – The index of the CUDA device for computations.
align_arguments (dict) – Additional arguments to be passed to the feature alignment trainer.

align(user_rkme: learnware.specification.regular.table.rkme.RKMETableSpecification)#

Train the align model using the RKME specifications from the user and the learnware.

Parameters: user_rkme (RKMETableSpecification) – The RKME specification from the user dataset.

predict(user_data: numpy.ndarray) → numpy.ndarray#

Predict the output for user data using the aligned model and learnware model.

Parameters: user_data (np.ndarray) – Input data for making predictions.
Returns: Predicted output from the learnware model after alignment.
Return type: np.ndarray

class learnware.reuse.HeteroMapAlignLearnware(learnware: Optional[learnware.learnware.base.Learnware] = None, mode: Optional[str] = None, cuda_idx=0, **align_arguments)#

Bases: learnware.reuse.align.AlignLearnware

HeteroMapAlignLearnware is a class designed for reusing learnware models with feature alignment and augmentation. It can handle both classification and regression tasks and supports fine-tuning on additional training data.

learnware#

The learnware model to be reused.

Type: Learnware

mode#

The mode of operation, either “classification” or “regression”.

Type: str

cuda_idx#

Index of the CUDA device to be used for computations.

Type: int

align_arguments#

Additional arguments for feature alignment.

Type: dict

__init__(learnware: Optional[learnware.learnware.base.Learnware] = None, mode: Optional[str] = None, cuda_idx=0, **align_arguments)#

Initialize the HeteroMapAlignLearnware with a learnware model, mode, CUDA device index, and alignment arguments.

Parameters

learnware (Learnware) – A learnware model used for initial predictions.
mode (str) – The mode of operation, either “regression” or “classification”.
cuda_idx (int) – The index of the CUDA device for computations.
align_arguments (dict) – Additional arguments to be passed to the feature alignment process.

align(user_rkme: learnware.specification.regular.table.rkme.RKMETableSpecification, x_train: Optional[numpy.ndarray] = None, y_train: Optional[numpy.ndarray] = None)#

Align the hetero learnware using the user RKME specification and labeled data.

Parameters

user_rkme (RKMETableSpecification) – The RKME specification from the user dataset.
x_train (ndarray) – Training data features.
y_train (ndarray) – Training data labels.

predict(user_data)#

Predict the output for user data using the feature aligner or the fine-tuned model.

Parameters: user_data (ndarray) – Input data for making predictions.
Returns: Predicted output from the model.
Return type: ndarray

Specification#

class learnware.specification.Specification(semantic_spec: Optional[dict] = None, stat_spec: Optional[Dict[str, learnware.specification.base.BaseStatSpecification]] = None)#

Bases: object

The specification interface, which manages the semantic specifications and statistical specifications

__init__(semantic_spec: Optional[dict] = None, stat_spec: Optional[Dict[str, learnware.specification.base.BaseStatSpecification]] = None)#

The initialization method

Parameters

semantic_spec (dict, optional) – The initiailzed semantic specification, by default None
stat_spec (Dict[str, BaseStatSpecification], optional) – The initiailzaed statistical specification, by default None

update_semantic_spec(semantic_spec: dict)#

Update semantic specification

Parameters: semantic_spec (dict) – The new sementic specifications

update_stat_spec(*args, **kwargs)#: Update the statistical specification by the way of ‘name’=’value’ or use class name as default name

get_stat_spec_by_name(name: str) → learnware.specification.base.BaseStatSpecification#

Get statistical specification by its name

Parameters: name (str) – The name of statistical specification
Returns: The corresponding statistical specification w.r.t name
Return type: BaseStatSpecification

class learnware.specification.BaseStatSpecification(type: str)#

Bases: object

The Statistical Specification Interface, which provide save and load method

__init__(type: str)#: initilize the type of stats specification :param type: the type of the stats specification :type type: str

generate_stat_spec(**kwargs)#: Construct statistical specification

save(filepath: str)#

Save the statistical specification into file in filepath

Parameters: filepath (str) – The saved file path

load(filepath: str)#

Load the statistical specification from file

Parameters: filepath (str) – The file path to load

Regular Specification#

class learnware.specification.RegularStatSpecification(type: str)#

Bases: learnware.specification.base.BaseStatSpecification

generate_stat_spec(**kwargs)#: Construct statistical specification

generate_stat_spec_from_data(**kwargs)#: Construct statistical specification from raw dataset - kwargs may include the feature, label and model - kwargs also can include hyperparameters of specific method for specifaction generation

class learnware.specification.RKMETableSpecification(gamma: float = 0.1, cuda_idx: Optional[int] = None)#

Bases: learnware.specification.regular.base.RegularStatSpecification

Reduced Kernel Mean Embedding (RKME) Specification

__init__(gamma: float = 0.1, cuda_idx: Optional[int] = None)#

Initializing RKME parameters.

Parameters

gamma (float) – Bandwidth in gaussian kernel, by default 0.1.
cuda_idx (int) – A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. None indicates automatically choose device

get_beta() → numpy.ndarray#

Move beta(RKME weights) back to memory accessible to the CPU.

Returns: A copy of beta in CPU memory.
Return type: np.ndarray

get_z() → numpy.ndarray#

Move z(RKME reduced set points) back to memory accessible to the CPU.

Returns: A copy of z in CPU memory.
Return type: np.ndarray

generate_stat_spec_from_data(X: numpy.ndarray, K: int = 100, step_size: float = 0.1, steps: int = 3, nonnegative_beta: bool = True, reduce: bool = True)#

Construct reduced set from raw dataset using iterative optimization.

Parameters

X (np.ndarray or torch.tensor) – Raw data in np.ndarray format.
K (int) – Size of the construced reduced set.
step_size (float) – Step size for gradient descent in the iterative optimization.
steps (int) – Total rounds in the iterative optimization.
nonnegative_beta (bool, optional) – True if weights for the reduced set are intended to be kept non-negative, by default False.
reduce (bool, optional) – Whether shrink original data to a smaller set, by default True

_init_z_by_kmeans(X: Union[numpy.ndarray, torch._VariableFunctionsClass.tensor], K: int)#

Intialize Z by kmeans clustering.

Parameters

X (np.ndarray or torch.tensor) – Raw data in np.ndarray format or torch.tensor format.
K (int) – Size of the construced reduced set.

_update_beta(X: Any, nonnegative_beta: bool = True)#

Fix Z and update beta using its closed-form solution.

Parameters

X (np.ndarray or torch.tensor) – Raw data in np.ndarray format or torch.tensor format.
nonnegative_beta (bool, optional) – True if weights for the reduced set are intended to be kept non-negative, by default False.

_update_z_vectorize(alpha: float, X: Any, step_size: float, batch_size=8)#

Fix beta and update Z using gradient descent. Unlike method _update_z, this method updates z by batches.

Parameters

alpha (int) – Normalization factor.
X (np.ndarray or torch.tensor) – Raw data in np.ndarray format or torch.tensor format.
step_size (float) – Step size for gradient descent.
batch_size (int) – To prevent exceeding GPU memory, process no more than batch_size at a time.

_update_z(alpha: float, X: Any, step_size: float)#

Fix beta and update Z using gradient descent.

Parameters

alpha (int) – Normalization factor.
X (np.ndarray or torch.tensor) – Raw data in np.ndarray format or torch.tensor format.
step_size (float) – Step size for gradient descent.

_inner_prod_with_X(X: Any) → float#

Compute the inner product between RKME specification and X

Parameters: X (np.ndarray or torch.tensor) – Raw data in np.ndarray format or torch.tensor format.
Returns: The inner product between RKME specification and X
Return type: float

_sampling_candidates(N: int) → numpy.ndarray#

Generate a large set of candidates as preparation for herding

Parameters: N (int) – The number of herding candidates.
Returns: The herding candidates.
Return type: np.ndarray

inner_prod(Phi2: learnware.specification.regular.table.rkme.RKMETableSpecification) → float#

Compute the inner product between two RKME specifications

Parameters: Phi2 (RKMETableSpecification) – The other RKME specification.
Returns: The inner product between two RKME specifications.
Return type: float

dist(Phi2: learnware.specification.regular.table.rkme.RKMETableSpecification, omit_term1: bool = False) → float#

Compute the Maximum-Mean-Discrepancy(MMD) between two RKME specifications

Parameters

Phi2 (RKMETableSpecification) – The other RKME specification.
omit_term1 (bool, optional) – True if the inner product of self with itself can be omitted, by default False.

herding(T: int) → numpy.ndarray#

Iteratively sample examples from an unknown distribution with the help of its RKME specification

Parameters: T (int) – Total iteration number for sampling.
Returns: A collection of examples which approximate the unknown distribution.
Return type: np.ndarray

save(filepath: str)#

Save the computed RKME specification to a specified path in JSON format.

Parameters: filepath (str) – The specified saving path.

load(filepath: str) → bool#

Load a RKME specification file in JSON format from the specified path.

Parameters: filepath (str) – The specified loading path.
Returns: True if the RKME is loaded successfully.
Return type: bool

class learnware.specification.RKMEImageSpecification(cuda_idx: Optional[int] = None, **kwargs)#

Bases: learnware.specification.regular.base.RegularStatSpecification

__init__(cuda_idx: Optional[int] = None, **kwargs)#

Initializing RKME Image specification’s parameters.

Parameters: cuda_idx (int) – A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. None indicates automatically choose device

generate_stat_spec_from_data(X: numpy.ndarray, K: int = 50, step_size: float = 0.01, steps: int = 100, resize: bool = True, sample_size: int = 5000, nonnegative_beta: bool = True, reduce: bool = True, verbose: bool = True, **kwargs)#

Construct reduced set from raw dataset using iterative optimization.

Parameters

X (np.ndarray or torch.tensor) – Raw data in [N, C, H, W] format.
K (int) – Size of the construced reduced set.
step_size (float) – Step size for gradient descent in the iterative optimization.
steps (int) – Total rounds in the iterative optimization.
resize (bool) – Whether to scale the image to the requested size, by default True.
sample_size (int) – Size of sampled set used to generate specification
nonnegative_beta (bool, optional) – True if weights for the reduced set are intended to be kept non-negative, by default False.
reduce (bool, optional) – Whether shrink original data to a smaller set, by default True
verbose (bool, optional) – Whether to print training progress, by default True

inner_prod(Phi2: learnware.specification.regular.image.rkme.RKMEImageSpecification) → float#

Compute the inner product between two RKME Image specifications

Parameters: Phi2 (RKMEImageSpecification) – The other RKME Image specification.
Returns: The inner product between two RKME Image specifications.
Return type: float

dist(Phi2: learnware.specification.regular.image.rkme.RKMEImageSpecification, omit_term1: bool = False) → float#

Compute the Maximum-Mean-Discrepancy(MMD) between two RKME Image specifications

Parameters

Phi2 (RKMEImageSpecification) – The other RKME specification.
omit_term1 (bool, optional) – True if the inner product of self with itself can be omitted, by default False.

herding(T: int) → numpy.ndarray#

Iteratively sample examples from an unknown distribution with the help of its RKME specification

Parameters: T (int) – Total iteration number for sampling.
Returns: A collection of examples which approximate the unknown distribution.
Return type: np.ndarray

save(filepath: str)#

Save the computed RKME Image specification to a specified path in JSON format.

Parameters: filepath (str) – The specified saving path.

load(filepath: str) → bool#

Load a RKME Image specification file in JSON format from the specified path.

Parameters: filepath (str) – The specified loading path.
Returns: True if the RKME is loaded successfully.
Return type: bool

class learnware.specification.RKMETextSpecification(gamma: float = 0.1, cuda_idx: Optional[int] = None)#

Bases: learnware.specification.regular.table.rkme.RKMETableSpecification

Reduced Kernel Mean Embedding (RKME) Specification for Text

__init__(gamma: float = 0.1, cuda_idx: Optional[int] = None)#

Initializing RKME parameters.

Parameters

gamma (float) – Bandwidth in gaussian kernel, by default 0.1.
cuda_idx (int) – A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. None indicates automatically choose device

generate_stat_spec_from_data(X: list, K: int = 100, step_size: float = 0.1, steps: int = 3, nonnegative_beta: bool = True, reduce: bool = True)#

Construct reduced set from raw dataset using iterative optimization.

Parameters

X (np.ndarray or torch.tensor) – Raw data in np.ndarray format.
K (int) – Size of the construced reduced set.
step_size (float) – Step size for gradient descent in the iterative optimization.
steps (int) – Total rounds in the iterative optimization.
nonnegative_beta (bool, optional) – True if weights for the reduced set are intended to be kept non-negative, by default False.
reduce (bool, optional) – Whether shrink original data to a smaller set, by default True

System Specification#

class learnware.specification.HeteroMapTableSpecification(gamma: float = 0.1, cuda_idx: Optional[int] = None)#

Bases: learnware.specification.system.base.SystemStatSpecification

Heterogeneous Map-Table Specification

__init__(gamma: float = 0.1, cuda_idx: Optional[int] = None)#

Initializing HeteroMapTableSpecification parameters.

Parameters

gamma (float) – Bandwidth in gaussian kernel, by default 0.1.
cuda_idx (int) – A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used.

get_z() → numpy.ndarray#

Move z(RKME reduced set points) back to memory accessible to the CPU.

Returns: A copy of z in CPU memory.
Return type: np.ndarray

get_beta() → numpy.ndarray#

Move beta(RKME weights weights) back to memory accessible to the CPU.

Returns: A copy of beta in CPU memory.
Return type: np.ndarray

generate_stat_spec_from_system(heter_embedding: numpy.ndarray, rkme_spec: learnware.specification.regular.table.rkme.RKMETableSpecification)#

Construct heterogeneous map-table specification from RKME specification and embedding genereated by heterogeneous market mapping.

Parameters

heter_embedding (np.ndarray) – Embedding genereated by the heterogeneous market mapping.
rkme_spec (RKMETableSpecification) – The RKME specification.

inner_prod(Embed2: learnware.specification.system.hetero_table.HeteroMapTableSpecification) → float#

Compute the inner product between two HeteroMapTableSpecifications

Parameters: Embed2 (HeteroMapTableSpecification) – The other HeteroMapTableSpecification.
Returns: The inner product between two HeteroMapTableSpecifications.
Return type: float

dist(Embed2: learnware.specification.system.hetero_table.HeteroMapTableSpecification, omit_term1: bool = False) → float#

Compute the Maximum-Mean-Discrepancy(MMD) between two HeteroMapTableSpecifications

Parameters

Phi2 (HeteroMapTableSpecification) – The other HeteroMapTableSpecification.
omit_term1 (bool, optional) – True if the inner product of self with itself can be omitted, by default False.

load(filepath: str) → bool#

Load a HeteroMapTableSpecification file in JSON format from the specified path.

Parameters: filepath (str) – The specified loading path.
Returns: True if the HeteroMapTableSpecification is loaded successfully.
Return type: bool

save(filepath: str) → bool#

Save the computed HeteroMapTableSpecification to a specified path in JSON format.

Parameters: filepath (str) – The specified saving path.

Model#

Base Model#

class learnware.model.BaseModel(input_shape: tuple, output_shape: tuple)#

Bases: object

Base interface tor model standard when user want to submit learnware to market.

__init__(input_shape: tuple, output_shape: tuple)#

The initialization method for base model

Parameters

input_shape (tuple) – The shape of input features, which must be given when inherit BaseModel, could be used for checking learnware
output_shape (tuple) – The shape of output prediction, which must be given when inherit BaseModel, could be used for checking learnware

predict(X: numpy.ndarray) → numpy.ndarray#

The prediction method for model in learnware, which will be checked when learnware is submitted into the market.

Parameters: X (Union[np.ndarray, torch.tensor]) – The features array for prediciton
Returns: The predictions array
Return type: Union[np.ndarray, torch.tensor]

finetune(X: numpy.ndarray, y: numpy.ndarray)#

The finetune method for continuing train the model searched by market

Parameters

X (Union[np.ndarray, torch.tensor]) – features for finetuning
y (np.ndarray) – labels for finetuning

Container#

class learnware.client.container.ModelContainer(model_config: dict, learnware_dirpath: str, build: bool = True)#

Bases: learnware.model.base.BaseModel

__init__(model_config: dict, learnware_dirpath: str, build: bool = True)#

The initialization method for base model

Parameters

input_shape (tuple) – The shape of input features, which must be given when inherit BaseModel, could be used for checking learnware
output_shape (tuple) – The shape of output prediction, which must be given when inherit BaseModel, could be used for checking learnware

init_and_setup_env()#: We must set input_shape and output_shape

predict(X)#

The prediction method for model in learnware, which will be checked when learnware is submitted into the market.

Parameters: X (Union[np.ndarray, torch.tensor]) – The features array for prediciton
Returns: The predictions array
Return type: Union[np.ndarray, torch.tensor]

finetune(X, y) → None#

The finetune method for continuing train the model searched by market

Parameters

X (Union[np.ndarray, torch.tensor]) – features for finetuning
y (np.ndarray) – labels for finetuning

class learnware.client.container.ModelCondaContainer(model_config: dict, learnware_dirpath: str, conda_env: Optional[str] = None, build: bool = True)#

Bases: learnware.client.container.ModelContainer

__init__(model_config: dict, learnware_dirpath: str, conda_env: Optional[str] = None, build: bool = True)#

The initialization method for base model

Parameters

input_shape (tuple) – The shape of input features, which must be given when inherit BaseModel, could be used for checking learnware
output_shape (tuple) – The shape of output prediction, which must be given when inherit BaseModel, could be used for checking learnware

predict(X)#

The prediction method for model in learnware, which will be checked when learnware is submitted into the market.

Parameters: X (Union[np.ndarray, torch.tensor]) – The features array for prediciton
Returns: The predictions array
Return type: Union[np.ndarray, torch.tensor]

finetune(X, y) → None#

The finetune method for continuing train the model searched by market

Parameters

X (Union[np.ndarray, torch.tensor]) – features for finetuning
y (np.ndarray) – labels for finetuning

class learnware.client.container.ModelDockerContainer(model_config: dict, learnware_dirpath: str, docker_container: Optional[object] = None, build: bool = True)#

Bases: learnware.client.container.ModelContainer

__init__(model_config: dict, learnware_dirpath: str, docker_container: Optional[object] = None, build: bool = True)#

_summary_

Parameters: build (bool, optional) – Whether to build the docker env, by default True

_install_environment(conda_env)#

Install environment of a learnware in docker container

Parameters: conda_env (str) – a new conda environment will be created with the given name
Raises: Exception – Lack of the environment configuration file.

_setup_env_and_metadata()#: setup env and set the input and output shape by communicating with docker

_init_env()#: create docker container according to the str

_remove_env()#: remove the docker container

fit(X, y)#: fit model by the communicating with docker

predict(X)#: predict model by the communicating with docker

finetune(X, y) → None#: finetune model by the communicating with docker

class learnware.client.container.LearnwaresContainer(learnwares: Union[List[learnware.learnware.base.Learnware], learnware.learnware.base.Learnware], cleanup=True, mode='conda', ignore_error=True)#

Bases: object

__init__(learnwares: Union[List[learnware.learnware.base.Learnware], learnware.learnware.base.Learnware], cleanup=True, mode='conda', ignore_error=True)#

The initializaiton method for base reuser

Parameters: learnware_list (List[Learnware]) – The learnware list to reuse and make predictions

API Reference

Contents

API Reference#

Market#

Organizer#

Searcher#

Checker#

Learnware#

Reuser#

Data Independent Reuser#

Data Dependent Reuser#

Aligned Learnware#

Specification#

Regular Specification#

System Specification#

Model#

Base Model#

Container#