nebulous/optimiser-utility-evaluator
Code Issues Proposed changes

cost performance indicators

Browse Source 
Change-Id: If89d229423dee74a6c0a5a61403209b4f7c4a6de
This commit is contained in:
Marta 2024-04-08 13:00:43 +02:00
parent 73a63a390d
commit 98474277fc
34 changed files with 1609 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
utility-evaluator/pom.xml
View File

@ -15,6 +15,7 @@
<description>First release of Utility Evaluator component</description>
<properties>
<java.version>17</java.version>
<jackson.version>2.16.1</jackson.version>
</properties>
<dependencies>
<dependency>
@ -36,6 +37,16 @@
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.6.1</version> <!-- or the latest version -->
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.ow2.proactive</groupId>
<artifactId>sal-common</artifactId>

5
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/UtilityEvaluatorApplication.java
View File

@ -4,14 +4,13 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/* This is the equvalent of the main class which starts the Utility Evaluator */
@SpringBootApplication
public class UtilityEvaluatorApplication {
public static void main(String[] args) {
SpringApplication.run(UtilityEvaluatorApplication.class, args);
// The application is listening to messages and sav
}
}

74
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/UtilityEvaluatorController.java
View File

@ -2,37 +2,67 @@ package eu.nebulous.utilityevaluator;
import org.ow2.proactive.sal.model.NodeCandidate;
import java.util.List;
import java.util.Optional;
import org.springframework.stereotype.Component;
import eu.nebulous.utilityevaluator.communication.activemq.message.FetchNodeCandidatesMessage;
import eu.nebulous.utilityevaluator.communication.exnconnector.ExnConnector;
import eu.nebulous.utilityevaluator.communication.exnconnector.PerformanceIndicatorSendingService;
import eu.nebulous.utilityevaluator.communication.sal.NodeCandidatesFetchingService;
import eu.nebulous.utilityevaluator.nodecandidates.NodeCandidateConverter;
import eu.nebulous.utilityevaluator.nodecandidates.NodeCandidateDTO;
import jline.internal.Log;
import eu.nebulous.utilityevaluator.converter.NodeCandidateConverter;
import eu.nebulous.utilityevaluator.model.Application;
import eu.nebulous.utilityevaluator.model.NodeCandidateDTO;
import eu.nebulous.utilityevaluator.model.VariableDTO;
import eu.nebulous.utilityevaluator.regression.SimpleCostRegression;
import eu.nebulouscloud.exn.core.Publisher;
import eu.nebulouscloud.exn.core.SyncedPublisher;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Component
@RequiredArgsConstructor
/* The main controlling component. It coordinates the work TODO*/
@Slf4j
//@Component
public class UtilityEvaluatorController {
private final NodeCandidatesFetchingService nodeCandidatesService;
//this is the main method of Utiliy Evaluator. It creates a .csv file with available Node Candidates
public Optional<String> createNodeCandidatesTensor(FetchNodeCandidatesMessage message){
Log.info("Creating Node Candidates tensor...");
List<NodeCandidate> nodeCandidates = nodeCandidatesService.getNodeCandidates(message.getCloudProviders());
//convert Node Candidates, possibly also filter (in the future)
List<NodeCandidateDTO> convertedNodeCandidates = NodeCandidateConverter.convertToDtoList(nodeCandidates);
String csv = NodeCandidateConverter.convertToCsv(convertedNodeCandidates);
return Optional.of(csv);
private NodeCandidatesFetchingService nodeCandidatesService;
private PerformanceIndicatorSendingService performanceIndicatorSendingService;
public UtilityEvaluatorController(SyncedPublisher nodeCandidatesGetter, Publisher performanceIndicatorPublisher){
this.nodeCandidatesService = new NodeCandidatesFetchingService(nodeCandidatesGetter);
this.performanceIndicatorSendingService = new PerformanceIndicatorSendingService(performanceIndicatorPublisher);
}
public Application createInitialCostPerformanceIndicators(Application application){
/*
* for each component of the application (that has variables), it should:
*
* convert them to DTO
* for types variables that are there, create a list of arguments to the regression
* create regression object
* save it back in the application
* send the parameters via ActiveMQ (maybe in the handler?)
*/
for (String component : application.getVariables().keySet()){
List<NodeCandidate> nodeCandidates = nodeCandidatesService.getNodeCandidatesViaMiddleware(application, component);
log.info("Number of Node Candidates: {}", nodeCandidates.size());
if (nodeCandidates.isEmpty()){
log.error("SAL returned empty list, it is not possible to create cost performance indicator");
continue;
}
List<NodeCandidateDTO> convertedNodeCandidates = NodeCandidateConverter.convertToDtoList(nodeCandidates);
List<VariableDTO> componentVariables = application.getVariables().get(component);
SimpleCostRegression regression = new SimpleCostRegression(component, convertedNodeCandidates, componentVariables);
application.getCostPerformanceIndicators().put(component, regression);
};
log.info("Creating regression for cost performance indicators has been successfully finished");
performanceIndicatorSendingService.sendPerformanceIndicators(application);
log.info("Performance indicators sent");
return application;
}
}

41
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/UtilityEvaluatorListener.java
View File

@ -1,41 +0,0 @@
package eu.nebulous.utilityevaluator;
import java.util.Optional;
import org.json.JSONObject;
import org.springframework.stereotype.Component;
import eu.nebulous.utilityevaluator.communication.activemq.message.FetchNodeCandidatesMessage;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component
@RequiredArgsConstructor
public class UtilityEvaluatorListener {
// This is an old component that is going to be removed after tests
private final UtilityEvaluatorController controller;
//@JmsListener(destination = "eu.nebulouscloud.dsl.general")
//@JmsListener(destination = "TestTopic")
public void handleGeneralApplicationMessage(JSONObject message) {
// Process the received message
log.info("Received message: {}", message.toString());
FetchNodeCandidatesMessage clearedMessage = new FetchNodeCandidatesMessage(message);
log.info("Cleared message: {}", clearedMessage.toString());
Optional<String> nodeCandidatesTensor = controller.createNodeCandidatesTensor(clearedMessage);
if (nodeCandidatesTensor.isPresent()){
log.info("Tensor successfully created");
// If needed, you can also send a response back to another queue or topic
//jmsTemplate.convertAndSend("eu.nebulouslcloud.optimizer.solver.tensor", new NodeCandidatesTensorMessage(clearedMessage.getApplicationID(), nodeCandidatesTensor.get()));
log.info("Tensor was passed via ActiveMQ");
}
else {
log.error("There was an error during creating the tensor");
}
}
}

55
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/exnconnector/DslGenericMessageHandler.java Normal file
View File

@ -0,0 +1,55 @@
package eu.nebulous.utilityevaluator.communication.exnconnector;
import eu.nebulous.utilityevaluator.UtilityEvaluatorController;
import eu.nebulous.utilityevaluator.model.Application;
import eu.nebulouscloud.exn.core.Context;
import eu.nebulouscloud.exn.core.Handler;
import eu.nebulouscloud.exn.core.Publisher;
import eu.nebulouscloud.exn.core.SyncedPublisher;
import java.util.Map;
import org.apache.qpid.protonj2.client.Message;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
@Slf4j
//@Component
public class DslGenericMessageHandler extends Handler {
// This class handles the dsl.generic message that contains all information needed for application deployment
// The example of this message is provided: TODO
private UtilityEvaluatorController controller;
private static final ObjectMapper mapper = new ObjectMapper();
public DslGenericMessageHandler(SyncedPublisher nodeCandidatesGetter, Publisher performanceIndicatorPublisher){
super();
this.controller = new UtilityEvaluatorController(nodeCandidatesGetter, performanceIndicatorPublisher);
}
@Override
public void onMessage(String key, String address, Map body, Message message, Context context) {
log.info("Received by custom handler {} => {} = {}", key,address,String.valueOf(body));
log.info("Body={}", body.toString());
//ObjectMapper objectMapper = new ObjectMapper();
//GenericDSLMessage genericDSLMessage = objectMapper.readValue(body.toString(), GenericDSLMessage.class);
//Application appFromMessage = new Application(genericDSLMessage);
JsonNode appMessage = mapper.valueToTree(body);
Application app = new Application(appMessage);
log.info("Application {}, with name {}, has variables: {}", app.getApplicationId(), app.getApplicationName(), app.getVariables().toString());
app = controller.createInitialCostPerformanceIndicators(app);
//todo: send back this cost performance indicators;
}
}

27
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/exnconnector/ExnConnector.java
View File

@ -9,8 +9,11 @@ import eu.nebulouscloud.exn.Connector;
import eu.nebulouscloud.exn.core.Consumer;
import eu.nebulouscloud.exn.handlers.ConnectorHandler;
import eu.nebulouscloud.exn.settings.StaticExnConfig;
import lombok.Getter;
import eu.nebulouscloud.exn.core.Context;
import eu.nebulouscloud.exn.core.Publisher;
import eu.nebulouscloud.exn.core.SyncedPublisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -28,12 +31,22 @@ public class ExnConnector {
String BROKER_PASSWORD;
public static final String GENERAL_APP_CREATION_MESSAGE_TOPIC = "eu.nebulouscloud.ui.dsl.generic.>";
public final GeneralMessageHandler generalHandler;
private static final String GENERAL_APP_CREATION_MESSAGE_TOPIC = "eu.nebulouscloud.ui.dsl.generic.>";
private final DslGenericMessageHandler genericDSLHandler;
private static final String PERFOMANCE_INDICATORS_TOPIC = "eu.nebulouscloud.optimiser.controller.ampl.performanceindicators";
@Getter
private final Publisher performanceIndicatorPublisher;
private static final String GET_NODE_CANDIDATES_TOPIC= "eu.nebulouscloud.exn.sal.nodecandidate.get";
@Getter
private final SyncedPublisher nodeCandidatesGetter;
public ExnConnector(GeneralMessageHandler handler) {
public ExnConnector() {
super();
this.generalHandler = handler;
this.performanceIndicatorPublisher = new Publisher("costPerformanceIndicators", PERFOMANCE_INDICATORS_TOPIC, true, true);
this.nodeCandidatesGetter = new SyncedPublisher("getNodeCandidates", GET_NODE_CANDIDATES_TOPIC, true, true);
this.genericDSLHandler = new DslGenericMessageHandler(nodeCandidatesGetter, performanceIndicatorPublisher);
init();
@ -43,8 +56,8 @@ public class ExnConnector {
Connector c = new Connector(
"utilityevaluator",
new MyConnectorHandler(),
List.of(),
List.of(new Consumer("ui_all", GENERAL_APP_CREATION_MESSAGE_TOPIC, generalHandler ,true,true)),
List.of(performanceIndicatorPublisher, nodeCandidatesGetter),
List.of(new Consumer("ui_generic_message", GENERAL_APP_CREATION_MESSAGE_TOPIC, genericDSLHandler ,true,true)),
false,
false,
new StaticExnConfig(

71
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/exnconnector/ExnListener.java
View File

@ -1,71 +0,0 @@
package eu.nebulous.utilityevaluator.communication.exnconnector;
import eu.nebulouscloud.exn.Connector;
import eu.nebulouscloud.exn.core.Consumer;
import eu.nebulouscloud.exn.core.Context;
import eu.nebulouscloud.exn.core.Handler;
import eu.nebulouscloud.exn.handlers.ConnectorHandler;
import eu.nebulouscloud.exn.settings.StaticExnConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import org.apache.qpid.protonj2.client.Message;
import java.util.List;
import java.util.Map;
@Component
public class ExnListener extends Handler {
private static final Logger log = LoggerFactory.getLogger(ExnListener.class);
private String topicName;
private String address;
private Connector connector;
public ExnListener(String topicName, String address, String brokerHost, int brokerPort, String username, String password) {
this.topicName = topicName;
this.address = address;
ConnectorHandler connectorHandler = new ConnectorHandler() {
@Override
public void onReady(Context context) {
log.info("Connector ready. Registering consumer for topic: {}", topicName);
// Register this handler as a consumer for the specified topic and address
context.registerConsumer(new Consumer(topicName, address, ExnListener.this, true));
}
};
// Initialize the connector with the connector handler and configuration
this.connector = new Connector("ui", connectorHandler, List.of(), List.of(), false, false,
new StaticExnConfig(brokerHost, brokerPort, username, password));
}
@Override
public void onMessage(String key, String address, Map body, Message message, Context context) {
log.info("Received message on topic {}: key={}, address={}, body={}", topicName, key, address, body);
// Implement custom message processing logic here
processMessage(key, address, body, message, context);
}
public void start() {
try {
connector.start();
log.info("ExnListener started for topic: {}", topicName);
} catch (Exception e) {
log.error("Error starting ExnListener: {}", e.getMessage());
}
}
public void stop() {
try {
connector.stop();
log.info("ExnListener stopped for topic: {}", topicName);
} catch (Exception e) {
log.error("Error stopping ExnListener: {}", e.getMessage());
}
}
protected void processMessage(String key, String address, Map body, Message message, Context context) {
// This method can be overridden in subclasses for custom message processing
}
}

61
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/exnconnector/GeneralMessageHandler.java
View File

@ -1,61 +0,0 @@
package eu.nebulous.utilityevaluator.communication.exnconnector;
import eu.nebulous.utilityevaluator.UtilityEvaluatorController;
import eu.nebulouscloud.exn.core.Context;
import eu.nebulouscloud.exn.core.Handler;
import java.util.Map;
import org.apache.qpid.protonj2.client.Message;
import org.apache.qpid.protonj2.client.exceptions.ClientException;
import org.json.JSONObject;
import org.springframework.stereotype.Component;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component
public class GeneralMessageHandler extends Handler {
public final UtilityEvaluatorController controller;
public GeneralMessageHandler(UtilityEvaluatorController controller){
super();
this.controller = controller;
}
@Override
public void onMessage(String key, String address, Map body, Message message, Context context) {
log.info("Received by custom handler {} => {} = {}", key,address,String.valueOf(body));
log.info("Received message: {}", message.toString());
log.info("Body={}", body.toString());
JSONObject jsonObject = new JSONObject(body);
Map<String, Object> map = jsonObject.toMap();
try {
String applicationId = message.subject();
//todo: transform the old code to get it in the right format
} catch (ClientException e) {
e.printStackTrace();
log.error(e.getMessage());
}
/*FetchNodeCandidatesMessage clearedMessage = new FetchNodeCandidatesMessage(message);
log.info("Cleared message: {}", clearedMessage.toString());
Optional<String> nodeCandidatesTensor = controller.createNodeCandidatesTensor(clearedMessage);
if (nodeCandidatesTensor.isPresent()){
log.info("Tensor successfully created");
// If needed, you can also send a response back to another queue or topic
//jmsTemplate.convertAndSend("eu.nebulouslcloud.optimizer.solver.tensor", new NodeCandidatesTensorMessage(clearedMessage.getApplicationID(), nodeCandidatesTensor.get()));
log.info("Tensor was passed via ActiveMQ");
}
else {
log.error("There was an error during creating the tensor");
}*/
}
}

5
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/exnconnector/NodeCandidatesMessageHandler.java Normal file
View File

@ -0,0 +1,5 @@
package eu.nebulous.utilityevaluator.communication.exnconnector;
public class NodeCandidatesMessageHandler {
}

120
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/exnconnector/PerformanceIndicatorSendingService.java Normal file
View File

@ -0,0 +1,120 @@
package eu.nebulous.utilityevaluator.communication.exnconnector;
import java.util.List;
import java.util.Map;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import eu.nebulous.utilityevaluator.model.Application;
import eu.nebulous.utilityevaluator.model.VariableDTO;
import eu.nebulous.utilityevaluator.regression.SimpleCostRegression;
import eu.nebulouscloud.exn.core.Publisher;
import io.micrometer.common.lang.NonNull;
import lombok.AllArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@AllArgsConstructor
public class PerformanceIndicatorSendingService {
@NonNull
private Publisher performanceIndicatorPublisher;
private static final ObjectMapper jsonMapper = new ObjectMapper();
/* Example of the message:
{
"performanceIndicators":
{
"name": "cost_pi_0",
"variables": "spec_components_0_traits_0_properties_cpu", "spec_components_0_traits_0_properties_ram",
"coefficientsName": "COEFFICIENTS_0",
"initialCoefficients": "COEFFICIENTS_0 := 1 0.1 2 0.3;"
},
{
"name": "cost_pi_1",
"variables": "spec_components_1_traits_0_properties_cpu",
"coefficientsName": "COEFFICIENTS_1",
"initialCoefficients": "COEFFICIENTS_1 := 1 0.1 2 0.3;"
}
}
*/
public void sendPerformanceIndicators(Application app){
ObjectNode msg = jsonMapper.createObjectNode();
ObjectNode performanceIndicators = msg.withObject("PerformanceIndicators");
for (String component: app.getCostPerformanceIndicators().keySet()){
String piName = "cost_pi"+component;
performanceIndicators.put("name", piName);
//array of variables
List<String> variableNames = app.getVariables().get(component).stream().map(var -> var.getName()).collect(Collectors.toList());
performanceIndicators.put("variables", variableNames.toString());
//coefficientsName
String coefficientsName = "COEFFICIENTS_"+component;
performanceIndicators.put("coefficientsName", coefficientsName);
//initial coefficients
String initialCoefficients = mapInitialCoefficientsToString(coefficientsName, app.getCostPerformanceIndicators().get(component).getCoefficients());
performanceIndicators.put("initialCoefficients", initialCoefficients);
log.info("Prepared performance indicator {}", performanceIndicators);
}
log.info("Message to be sent: {}", msg);
performanceIndicatorPublisher.send(jsonMapper.convertValue(msg, Map.class), app.getApplicationId(), true);
}
private String mapInitialCoefficientsToString(String coefficientsName, double[] values) {
StringJoiner joiner = new StringJoiner(" ");
joiner.add(coefficientsName + " :=");
for (double num : values) {
joiner.add(Double.toString(num));
}
return joiner.toString();
}
/*
private String mapInitialCoefficientsToString(Map<String, double[]> map) {
StringJoiner joiner = new StringJoiner(" ");
map.forEach((componentName, value) -> {
joiner.add("COEFFICIENTS_"+componentName + " :=");
for (double num : value) {
joiner.add(Double.toString(num));
}
});
return joiner.toString();
}*/
/*public void sendAMPL() {
String ampl = AMPLGenerator.generateAMPL(this);
ObjectNode msg = jsonMapper.createObjectNode();
msg.put("ObjectiveFunction", getObjectiveFunction());
ObjectNode constants = msg.withObject("Constants");
// Define initial values for constant utility functions:
// "Constants" : {
// <constant utility function name> : {
// "Variable" : <AMPL Variable Name>
// "Value" : <value at the variable's path in original KubeVela>
// }
// }
for (final JsonNode function : originalAppMessage.withArray(utility_function_path)) {
if (!(function.get("type").asText().equals("constant")))
continue;
// NOTE: for a constant function, we rely on the fact that the
// function body is a single variable defined in the "Variables"
// section and pointing to KubeVela, and the
// `functionExpressionVariables` array contains one entry.
JsonNode variable = function.withArray("/expression/variables").get(0);
String variableName = variable.get("value").asText();
JsonPointer path = kubevelaVariablePaths.get(variableName);
JsonNode value = originalKubevela.at(path);
ObjectNode constant = constants.withObject(function.get("name").asText());
constant.put("Variable", variableName);
constant.set("Value", value);
}
log.info("Sending AMPL file to solver", keyValue("amplMessage", msg), keyValue("appId", UUID));
Main.logFile("to-solver-" + getUUID() + ".json", msg.toString());
Main.logFile("to-solver-" + getUUID() + ".ampl", ampl);
}*/
}

77
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/sal/NodeCandidatesFetchingService.java
View File

@ -1,5 +1,7 @@
package eu.nebulous.utilityevaluator.communication.sal;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@ -8,38 +10,95 @@ import org.ow2.proactive.sal.model.NodeCandidate;
import org.ow2.proactive.sal.model.Requirement;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.nebulous.utilityevaluator.communication.exnconnector.ExnConnector;
import eu.nebulous.utilityevaluator.communication.sal.error.ProactiveClientException;
import eu.nebulous.utilityevaluator.external.KubevelaAnalyzer;
import eu.nebulous.utilityevaluator.model.Application;
import eu.nebulouscloud.exn.core.SyncedPublisher;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Component
//@Component
public class NodeCandidatesFetchingService {
private static final int NUMBER_OF_REPEATS_FOR_NODE_CANDIDATES = 120;
private static final int DELAY_BETWEEN_REQUESTS = 5000;
private final ProactiveConnector proactiveClientConnectorService;
@NonNull
private SyncedPublisher nodeCandidatesConnector;
private static final ObjectMapper mapper = new ObjectMapper();
//private final ProactiveConnector proactiveClientConnectorService;
/*public NodeCandidatesFetchingService(ProactiveClientProperties properties){
ProactiveConnector connector = new ProactiveConnector(properties);
this.proactiveClientConnectorService = connector;
}*/
//https://gitlab.ow2.org/melodic/melodic-upperware/-/tree/morphemic-rc4.0/cp_generator/src/main/java/eu/paasage/upperware/profiler/generator/communication/impl
//https://gitlab.ow2.org/melodic/melodic-upperware/-/tree/morphemic-rc4.0/cp_generator/src/main/java/eu/paasage/upperware/profiler/generator/communication/impl
public List<NodeCandidate> getNodeCandidates(Map<String,String> cloudProviders){
public List<NodeCandidate> getNodeCandidatesViaMiddleware(Application app, String componentId){
/*generate requirements (based on kubevela), and providers,
* call SAL via EXN Middleware
* get node candidates
* */
Map<String, List<Requirement>> requirements = KubevelaAnalyzer.getRequirements(app.getKubevela());
Map<String, Object> message = new HashMap();
try {
message = Map.of("metaData", Map.of("user", "admin"), "body", mapper.writeValueAsString(requirements));
log.info("Sending message to SAL: {}", message);
} catch (JsonProcessingException e) {
log.error("There was an error during converting message {}", e);
e.printStackTrace();
}
//Map<String, Object> message = Map.of("metaData", Map.of("user", "admin"), "body", "[]");
Map<String, Object> response = nodeCandidatesConnector.sendSync(message, app.getApplicationId(), null, false);
log.info("Received a response");
JsonNode payload = extractPayloadFromExnResponse(response, app.getApplicationId(), "getNodeCandidates");
log.info("Correctly return SAL response for component {}", componentId);
return Arrays.asList(mapper.convertValue(payload, NodeCandidate[].class));
}
//copied from Optimizer Controller: https://opendev.org/nebulous/optimiser-controller/src/branch/master/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousApp.java
private static JsonNode extractPayloadFromExnResponse(Map<String, Object> responseMessage, String appID, String caller) {
JsonNode response = mapper.valueToTree(responseMessage);
String salRawResponse = response.at("/body").asText(); // it's already a string, asText() is for the type system
JsonNode metadata = response.at("/metaData");
JsonNode salResponse = mapper.missingNode(); // the data coming from SAL
try {
salResponse = mapper.readTree(salRawResponse);
}
catch (JsonProcessingException e) {
log.error("Could not read message body as JSON: body = '{}', for app: {}", salRawResponse, appID, e);
return mapper.missingNode();
}
if (!metadata.at("/status").asText().startsWith("2")) {
// we only accept 200, 202, numbers of that nature
log.error("exn-middleware-sal request failed with error code '{}' and message '{}'", metadata.at("/status"), salResponse.at("/message").asText());
return mapper.missingNode();
}
return salResponse;
}
//old method used for SAL
private List<NodeCandidate> getNodeCandidates(Map<String,String> cloudProviders){
List<Requirement> providerRequirements = convertProviderRequirements(cloudProviders);
return findNodeCandidates(providerRequirements);
}
//to be deleted
private List<Requirement> convertProviderRequirements(Map<String,String> cloudProviders){
//todo: filter based on the chosen cloud providers
return List.of();
}
//old method used to connect to SAL directly
private List<NodeCandidate> findNodeCandidates(List<Requirement> requirements) {
List<NodeCandidate> nodeCandidates = new LinkedList<>();
boolean isAnyAsyncNodeCandidatesProcessesInProgress = true;
@ -47,14 +106,14 @@ public class NodeCandidatesFetchingService {
try {
while (isAnyAsyncNodeCandidatesProcessesInProgress && (requestNo < NUMBER_OF_REPEATS_FOR_NODE_CANDIDATES)) {
log.info("Checking if nodeCandidates downlaod process is finished. Trye: {}", requestNo);
isAnyAsyncNodeCandidatesProcessesInProgress = proactiveClientConnectorService.isAnyAsyncNodeCandidatesProcessesInProgress();
//isAnyAsyncNodeCandidatesProcessesInProgress = proactiveClientConnectorService.isAnyAsyncNodeCandidatesProcessesInProgress();
Thread.sleep(DELAY_BETWEEN_REQUESTS);
requestNo++;
}
if (isAnyAsyncNodeCandidatesProcessesInProgress) {
throw new RuntimeException("NodeCandidates are not yet present inside proactive scheduler");
}
nodeCandidates = proactiveClientConnectorService.fetchNodeCandidates(requirements);
//nodeCandidates = proactiveClientConnectorService.fetchNodeCandidates(requirements);
} catch (InterruptedException e1) {
e1.printStackTrace();
} catch (ProactiveClientException e2) {

2
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/sal/ProactiveConnector.java
View File

@ -30,7 +30,7 @@ import java.util.*;
@Slf4j
@Service
//@Service
public class ProactiveConnector {
private static final String SESSION_HEADER = "sessionid";

48
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/nodecandidates/NodeCandidateConverter.java → utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/converter/NodeCandidateConverter.java
View File

@ -1,11 +1,15 @@
package eu.nebulous.utilityevaluator.nodecandidates;
package eu.nebulous.utilityevaluator.converter;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.ow2.proactive.sal.model.NodeCandidate;
import eu.nebulous.utilityevaluator.model.NodeCandidateDTO;
import eu.nebulous.utilityevaluator.model.VariableDTO;
import lombok.extern.slf4j.Slf4j;
//static class that converts Node Candidates from SAL to NodeCandidatesDTO, which are then directly saved in node Candidates tensor
@Slf4j
public class NodeCandidateConverter {
public static final String CSV_HEADER = "id;gpu;cpu;ram;location;latitude;longitude;provider;type;price\n";
@ -61,4 +65,44 @@ public class NodeCandidateConverter {
nodeCandidate.getPrice());
}
//TODO: this method should also encode other fields of NodeCandidates: GPU, providertype, location but we need to have them as variables.
//only for variables that are used
public static double[][] convertListToDoubleArray(List<NodeCandidateDTO> nodeList, List<VariableDTO> variables) {
int size = nodeList.size();
double[][] dataArray = new double[size][];
for (int i = 0; i < size; i++) {
NodeCandidateDTO node = nodeList.get(i);
List<Integer> usedNodeParameters = new ArrayList<>();
for (VariableDTO variable : variables){
switch (variable.getType()){
case CPU:
usedNodeParameters.add(node.getCpu());
break;
case RAM:
usedNodeParameters.add(Long.valueOf(node.getRam()).intValue());
break;
default:
log.info("Variable type {} is not usable in cost performance indicators", variable.getType());
break;
}
}
double[] data = new double[usedNodeParameters.size()];
for (int j = 0; j < usedNodeParameters.size(); j++){
data[j]=usedNodeParameters.get(j);
}
/*{
node.getCpu(),
node.getGpu(),
node.getRam(),
node.getLatitude(),
node.getLongitude()
};*/
dataArray[i] = data;
}
return dataArray;
}
}

98
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/converter/VariableConverter.java Normal file
View File

@ -0,0 +1,98 @@
package eu.nebulous.utilityevaluator.converter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.JsonNode;
import eu.nebulous.utilityevaluator.model.VariableDTO;
import eu.nebulous.utilityevaluator.model.VariableType;
import eu.nebulous.utilityevaluator.model.message.Variable;
import lombok.extern.slf4j.Slf4j;
/*
* This class is responsible for converting variables from generic.dsl message to a map < String, List<VariableDTO>>,
* where the key is the component name and the value is the list of variables related to this component
*/
@Slf4j
public class VariableConverter{
public static Map<String, List<VariableDTO>> convertAndGroupVariables(JsonNode variables){
Map<String, List<VariableDTO>> mapOfVariables = new HashMap<>();
List<String> componentNames = new ArrayList<>();
if (!variables.isArray()) {
log.warn("No variables have been defined, is not possible to perform any optimization");
return mapOfVariables;
}
for (JsonNode var : variables) {
//var.get("key").asText(); //to get the name of the variable
//var.get("meaning").asText(); //to get the type
//to get the path
String component = getPrefix(var.get("path").asText());
log.info("Component: {}", component);
String meaning = getLastPart(var.get("meaning").asText());
log.info("meaning: {}", meaning);
VariableDTO variableDTO = new VariableDTO(var.get("key").asText(), component, meaning);
//if it is a first variable related to this component
if (!mapOfVariables.containsKey(component)) {
componentNames.add(component);
ArrayList<VariableDTO> variablesList = new ArrayList<>();
variablesList.add(variableDTO);
mapOfVariables.put(component, variablesList);
log.info("Adding new variable: {} for component: {}", variableDTO.getName(), variableDTO.getComponentName());
}
else {
mapOfVariables.get(component).add(variableDTO);
}
}
return mapOfVariables;
}
private static String getPrefix(String str) {
int endIndex = str.indexOf('/', "/spec/components/".length());
if (endIndex == -1) {
return str;
}
return str.substring(0, endIndex);
}
private static String getLastPart(String input) {
String[] parts = input.split("\\.");
return parts[parts.length - 1];
}
//old message, to be deleted
public static Map<String, List<VariableDTO>> convertAndGroupVariables(List<Variable> variables){
Map<String, List<VariableDTO>> mapOfVariables = new HashMap<>();
List<String> componentNames = new ArrayList<>();
for (Variable v: variables){
String prefix = getPrefix(v.getPath());
VariableDTO variableDTO = new VariableDTO(v.getKey(), prefix, v.getMeaning());
if (!mapOfVariables.containsKey(prefix)) { //if it is a first variable related to this component
componentNames.add(prefix);
ArrayList<VariableDTO> variablesList = new ArrayList<>();
variablesList.add(variableDTO);
mapOfVariables.put(prefix, variablesList);
}
mapOfVariables.get(prefix).add(variableDTO);
}
return mapOfVariables;
}
}

208
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/external/KubevelaAnalyzer.java vendored Normal file
View File

@ -0,0 +1,208 @@
package eu.nebulous.utilityevaluator.external;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.ow2.proactive.sal.model.AttributeRequirement;
import org.ow2.proactive.sal.model.OperatingSystemFamily;
import org.ow2.proactive.sal.model.Requirement;
import org.ow2.proactive.sal.model.RequirementOperator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* A collection of methods to extract node requirements from KubeVela files.
*/
public class KubevelaAnalyzer {
private static Logger log = LoggerFactory.getLogger(KubevelaAnalyzer.class);
private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
/**
* Given a KubeVela file, extract how many nodes to deploy for each
* component. Note that this can be zero when the component should not be
* deployed at all. This can happen for example when there is a cloud and
* an edge version of the component and only one of them should run.<p>
*
* We currently look for the following component trait:
*
* <pre>{@code
* traits:
* - type: scaler
* properties:
* replicas: 2
* }</pre>
*
* If this trait is not found for a component, its count will be 1.
*
* @param kubevela the parsed KubeVela file.
* @return A map from component name to number of instances to generate.
*/
public static Map<String, Integer> getNodeCount(JsonNode kubevela) {
Map<String, Integer> result = new HashMap<>();
ArrayNode components = kubevela.withArray("/spec/components");
for (final JsonNode c : components) {
result.put(c.get("name").asText(), 1); // default value; might get overwritten
for (final JsonNode t : c.withArray("/traits")) {
if (t.at("/type").asText().equals("scaler")
&& t.at("/properties/replicas").canConvertToExactIntegral())
{
result.put(c.get("name").asText(),
t.at("/properties/replicas").asInt());
}
}
}
return result;
}
/**
* Extract node count from a KubeVela file.
*
* @see #getNodeCount(JsonNode)
* @param kubevela The KubeVela file, as a YAML string.
* @return A map from component name to number of instances to generate.
* @throws JsonProcessingException if the argument does not contain valid YAML.
*/
public static Map<String, Integer> getNodeCount(String kubevela) throws JsonProcessingException {
return getNodeCount(parseKubevela(kubevela));
}
/**
* Extract node requirements from a KubeVela file in a form we can send to
* the SAL `findNodeCandidates` endpoint. <p>
*
* We read the following attributes for each component:
*
* - `properties.cpu`, `properties.requests.cpu`: round up to next integer
* and generate requirement `hardware.cores`
*
* - `properties.memory`, `properties.requests.memory`: Handle "200Mi",
* "0.2Gi" and bare number, convert to MB and generate requirement
* `hardware.memory`
*
* Notes:<p>
*
* - We add the requirement that OS family == Ubuntu.<p>
*
* - For the first version, we specify all requirements as "greater or
* equal", i.e., we might not find precisely the node candidates that
* are asked for. <p>
*
* - Related, KubeVela specifies "cpu" as a fractional value, while SAL
* wants the number of cores as a whole number. We round up to the
* nearest integer and ask for "this or more" cores, since we might end
* up with needing, e.g., 3 cores, which is not a configuration commonly
* provided by cloud providers. <p>
*
* @param kubevela the parsed KubeVela file.
* @return a map of component name to (potentially empty, except for OS
* family) list of requirements for that component. No requirements mean
* any node will suffice.
*/
public static Map<String, List<Requirement>> getRequirements(JsonNode kubevela) {
Map<String, List<Requirement>> result = new HashMap<>();
ArrayNode components = kubevela.withArray("/spec/components");
for (final JsonNode c : components) {
String componentName = c.get("name").asText();
ArrayList<Requirement> reqs = new ArrayList<>();
reqs.add(new AttributeRequirement("image", "operatingSystem.family",
RequirementOperator.IN, OperatingSystemFamily.UBUNTU.toString()));
JsonNode cpu = c.at("/properties/cpu");
if (cpu.isMissingNode()) cpu = c.at("/properties/resources/requests/cpu");
if (!cpu.isMissingNode()) {
// KubeVela has fractional core /cpu requirements, and the
// value might be given as a string instead of a number, so
// parse string in all cases.
double kubevela_cpu = -1;
try {
kubevela_cpu = Double.parseDouble(cpu.asText());
} catch (NumberFormatException e) {
log.warn("CPU spec in {} is not a number, value seen is {}",
componentName, cpu.asText());
}
long sal_cores = Math.round(Math.ceil(kubevela_cpu));
if (sal_cores > 0) {
reqs.add(new AttributeRequirement("hardware", "cores",
RequirementOperator.GEQ, Long.toString(sal_cores)));
} else {
// floatValue returns 0.0 if node is not numeric
log.warn("CPU of component {} is 0 or not a number, value seen is {}",
componentName, cpu.asText());
}
}
JsonNode memory = c.at("/properties/memory");
if (memory.isMissingNode()) cpu = c.at("/properties/resources/requests/memory");
if (!memory.isMissingNode()) {;
String sal_memory = memory.asText();
if (sal_memory.endsWith("Mi")) {
sal_memory = sal_memory.substring(0, sal_memory.length() - 2);
} else if (sal_memory.endsWith("Gi")) {
sal_memory = String.valueOf(Integer.parseInt(sal_memory.substring(0, sal_memory.length() - 2)) * 1024);
} else if (!memory.isNumber()) {
log.warn("Unsupported memory specification in component {} :{} (wanted 'Mi' or 'Gi') ",
componentName,
memory.asText());
sal_memory = null;
}
// Fall-through: we rewrote the KubeVela file and didn't add
// the "Mi" suffix, but it's a number
if (sal_memory != null) {
reqs.add(new AttributeRequirement("hardware", "memory",
RequirementOperator.GEQ, sal_memory));
}
}
for (final JsonNode t : c.withArray("/traits")) {
// TODO: Check for node affinity / geoLocation / country /
// node type (edge or cloud)
}
// Finally, add requirements for this job to the map
result.put(componentName, reqs);
}
return result;
}
/**
* Extract node requirements from a KubeVela file.
*
* @see #getRequirements(JsonNode)
* @param kubevela The KubeVela file, as a YAML string.
* @return a map of component name to (potentially empty, except for OS
* family) list of requirements for that component. No requirements mean
* any node will suffice.
* @throws JsonProcessingException if kubevela does not contain valid YAML.
*/
public static Map<String, List<Requirement>> getRequirements(String kubevela) throws JsonProcessingException {
return getRequirements(parseKubevela(kubevela));
}
/**
* Convert YAML KubeVela into a parsed representation.
*
* @param kubevela The KubeVela YAML.
* @return A parsed representation of the KubeVela file, or null for a parse error.
* @throws JsonProcessingException if kubevela does not contain valid YAML.
*/
public static JsonNode parseKubevela(String kubevela) throws JsonProcessingException {
return yamlMapper.readTree(kubevela);
}
/**
* Convert the parsed representation of a KubeVela file to yaml.
*
* @param kubevela The KubeVela parsed file.
* @return A YAML representation of the KubeVela file.
* @throws JsonProcessingException if YAML cannot be generated from kubevela.
*/
public static String generateKubevela(JsonNode kubevela) throws JsonProcessingException {
return yamlMapper.writeValueAsString(kubevela);
}
}

70
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/model/Application.java Normal file
View File

@ -0,0 +1,70 @@
package eu.nebulous.utilityevaluator.model;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import eu.nebulous.utilityevaluator.converter.VariableConverter;
import eu.nebulous.utilityevaluator.model.message.GenericDSLMessage;
import eu.nebulous.utilityevaluator.regression.SimpleCostRegression;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Getter
@RequiredArgsConstructor
@ToString
public class Application {
@NonNull
private String applicationId;
private String applicationName;
@NonNull
private JsonNode kubevela;
private List<String> chosenProviders;
@NonNull
private Map<String, List<VariableDTO>> variables;
@Setter
private Map<String, SimpleCostRegression> costPerformanceIndicators;
/** Location of the kubevela yaml file in the app creation message (String) */
private static final JsonPointer KUBEVELA_PATH = JsonPointer.compile("/content");
/** Location of the variables (optimizable locations) of the kubevela file
* in the app creation message. (Array of objects) */
private static final JsonPointer VARIABLES_PATH = JsonPointer.compile("/variables");
/** Locations of the UUID and name in the app creation message (String) */
private static final JsonPointer UUID_PATH = JsonPointer.compile("/uuid");
private static final JsonPointer NAME_PATH = JsonPointer.compile("/title");
/** Location of the variables (optimizable locations) of the kubevela file
* in the app creation message. (Array of objects) */
private static final JsonPointer PROVIDERS_PATH = JsonPointer.compile("/resources");
public Application (JsonNode appMessage) {
try {
this.kubevela = appMessage.at(KUBEVELA_PATH);
this.applicationId = appMessage.at(UUID_PATH).textValue();
this.applicationName = appMessage.at(NAME_PATH).textValue();
JsonNode variables = appMessage.at(VARIABLES_PATH);
this.variables = VariableConverter.convertAndGroupVariables(variables);
log.info("Application message successfully parsed");
} catch (Exception e) {
log.error("Could not read app creation message", e);
}
}
/*public Application(GenericDSLMessage message){
this.applicationId = message.getUuid();
this.kubevela = message.getContent();
this.applicationName = message.getTitle();
message.getVariables();
message.getResources();
}*/
}

2
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/nodecandidates/NodeCandidateDTO.java → utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/model/NodeCandidateDTO.java
View File

@ -1,4 +1,4 @@
package eu.nebulous.utilityevaluator.nodecandidates;
package eu.nebulous.utilityevaluator.model;
import org.ow2.proactive.sal.model.NodeCandidate;

17
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/model/VariableDTO.java Normal file
View File

@ -0,0 +1,17 @@
package eu.nebulous.utilityevaluator.model;
import lombok.Getter;
@Getter
public class VariableDTO {
private String name;
private String componentName;
private VariableType type;
public VariableDTO(String name, String componentName, String meaning){
this.name = name;
this.componentName = componentName;
this.type = VariableType.fromValue(meaning);
}
}

26
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/model/VariableType.java Normal file
View File

@ -0,0 +1,26 @@
package eu.nebulous.utilityevaluator.model;
public enum VariableType {
CPU ("cpu"),
RAM ("memory"),
LOCATION ("location"),
STORAGE("storage"),
REPLICAS ("replicas");
private final String value;
VariableType (String value){
this.value = value;
}
public static VariableType fromValue(String value) {
for (VariableType enumValue : VariableType.values()) {
if (enumValue.value.equals(value)) {
return enumValue;
}
}
throw new IllegalArgumentException("No enum constant with value: " + value);
}
}

2
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/activemq/message/FetchNodeCandidatesMessage.java → utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/model/message/FetchNodeCandidatesMessage.java
View File

@ -1,4 +1,4 @@
package eu.nebulous.utilityevaluator.communication.activemq.message;
package eu.nebulous.utilityevaluator.model.message;
import java.io.Serializable;
import java.util.HashMap;

132
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/model/message/GenericDSLMessage.java Normal file
View File

@ -0,0 +1,132 @@
package eu.nebulous.utilityevaluator.model.message;
import java.util.List;
import lombok.Getter;
@Getter
public class GenericDSLMessage {
private String title;
private String uuid;
private String content;
private List<Variable> variables;
private List<Resource> resources;
private List<Template> templates;
private List<Parameter> parameters;
private List<Metric> metrics;
private SloViolations sloViolations;
private List<UtilityFunction> utilityFunctions;
private List<Object> environmentVariables;
}
class Value {
private int lower_bound;
private int higher_bound;
// Getters and setters
}
@Getter
class Resource {
private String uuid;
private String title;
private String platform;
private boolean enabled;
// Getters and setters
}
class Template {
private String id;
private String type;
private int minValue;
private int maxValue;
private String unit;
// Getters and setters
}
class Parameter {
private String name;
private String template;
// Getters and setters
}
class Metric {
private String type;
private String name;
private String formula;
private boolean isWindowInput;
private Input input;
private boolean isWindowOutput;
private Output output;
private List<String> arguments;
// Getters and setters
}
class Input {
private String type;
private int interval;
private String unit;
// Getters and setters
}
class Output {
private String type;
private int interval;
private String unit;
// Getters and setters
}
class SloViolations {
private String nodeKey;
private boolean isComposite;
private String condition;
private boolean not;
private List<SloViolation> children;
// Getters and setters
}
class SloViolation {
private String nodeKey;
private boolean isComposite;
private String metricName;
private String operator;
private String value;
// Getters and setters
}
class UtilityFunction {
private String name;
private String type;
private Expression expression;
// Getters and setters
}
class Expression {
private String formula;
private List<VariableValue> variables;
// Getters and setters
}
class VariableValue {
private String name;
private String value;
// Getters and setters
}

2
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/communication/activemq/message/NodeCandidatesTensorMessage.java → utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/model/message/NodeCandidatesTensorMessage.java
View File

@ -1,4 +1,4 @@
package eu.nebulous.utilityevaluator.communication.activemq.message;
package eu.nebulous.utilityevaluator.model.message;
import java.io.Serializable;

17
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/model/message/Variable.java Normal file
View File

@ -0,0 +1,17 @@
package eu.nebulous.utilityevaluator.model.message;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class Variable {
private String key;
private String path;
private String type;
private String meaning;
private Value value;
}

47
utility-evaluator/src/main/java/eu/nebulous/utilityevaluator/regression/SimpleCostRegression.java Normal file
View File

@ -0,0 +1,47 @@
package eu.nebulous.utilityevaluator.regression;
import java.util.List;
import org.apache.commons.math3.stat.regression.OLSMultipleLinearRegression;
import eu.nebulous.utilityevaluator.converter.NodeCandidateConverter;
import eu.nebulous.utilityevaluator.model.NodeCandidateDTO;
import eu.nebulous.utilityevaluator.model.VariableDTO;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
/* TODO: comments
this class looks good, except that the coefficient should be filtered based on the variables
*/
@Slf4j
public class SimpleCostRegression {
private String componentName; //indicates for which component this cost regression is done
private OLSMultipleLinearRegression regression;
@Getter
private double[] coefficients; //the results of the regression
public SimpleCostRegression(String componentName, List<NodeCandidateDTO> nodeCandidates, List<VariableDTO> variables){
this.componentName = componentName;
this.regression = new OLSMultipleLinearRegression();
regression.newSampleData(convertPricesToArray(nodeCandidates), NodeCandidateConverter.convertListToDoubleArray (nodeCandidates, variables));
log.info("Data for component {} was loaded", componentName);
this.coefficients = regression.estimateRegressionParameters();
log.info("Coefficients: {}", coefficients);
}
private static double[] convertPricesToArray(List<NodeCandidateDTO> nodeCandidates) {
int size = nodeCandidates.size();
double[] pricesArray = new double[size];
for (int i = 0; i < size; i++) {
NodeCandidateDTO node = nodeCandidates.get(i);
pricesArray[i] = node.getPrice();
}
return pricesArray;
}
}

24
utility-evaluator/src/test/java/eu/nebulous/utilityevaluator/UtilityEvaluatorApplicationTests.java
View File

@ -1,5 +1,6 @@
package eu.nebulous.utilityevaluator;
import org.json.simple.parser.JSONParser;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.ow2.proactive.sal.model.Cloud;
@ -12,15 +13,20 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.jms.core.JmsTemplate;
import eu.nebulous.utilityevaluator.communication.activemq.message.FetchNodeCandidatesMessage;
import eu.nebulous.utilityevaluator.nodecandidates.NodeCandidateConverter;
import eu.nebulous.utilityevaluator.nodecandidates.NodeCandidateDTO;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.nebulous.utilityevaluator.converter.NodeCandidateConverter;
import eu.nebulous.utilityevaluator.model.NodeCandidateDTO;
import eu.nebulous.utilityevaluator.model.message.FetchNodeCandidatesMessage;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@ -31,6 +37,18 @@ class UtilityEvaluatorApplicationTests {
@MockBean
private JmsTemplate jmsTemplate;
//@Test
void testConvertingNodeCandidates(){
String filename = "/Users/martarozanska/nebulous/git/optimiser-utility-evaluator/utility-evaluator/src/test/java/resources/response-all-clouds.json";
File fileNC = new File(filename);
ObjectMapper mapper = new ObjectMapper();
//List<NodeCandidate> nodeCandidates = mapper.readValue(fileNC, );
}
//@Test
void testNodeCandidatesConverter() {
// Arrange

25
utility-evaluator/src/test/java/resources/2024-02-27T12_31_47.372531--to-solver-419c5ac2-e8cb-4115-8aa1-27d41ba0a08e.ampl Normal file
View File

@ -0,0 +1,25 @@
# AMPL file for application 'Mercabana Intralogistics' with id 419c5ac2-e8cb-4115-8aa1-27d41ba0a08e
# Variables
var spec_components_0_traits_0_properties_replicas integer >= 1, <= 8;
# Metrics. Note that we only emit metrics that are in use. Values will be provided by the solver.
## Raw metrics
## Composite metrics
param mean_job_process_time; # mean_job_process_time
# Constants
param currentReplicas;
# Performance indicators = composite metrics that have at least one variable in their formula
# Performance indicator formulas
# TBD: cost parameters - for all components! and use of node-candidates tensor
# Utility functions
maximize utilityfunction :
-1*((mean_job_process_time*currentReplicas)/spec_components_0_traits_0_properties_replicas);
# Default utility function: specified in message to solver
# Constraints. For constraints we don't have name from GUI, must be created

32
utility-evaluator/src/test/java/resources/2024-03-06T15_31_57.251674--to-solver-419c5ac2-e8cb-4115-8aa1-27d41ba0a08e.ampl Normal file
View File

@ -0,0 +1,32 @@
# AMPL file for application 'Mercabana Intralogistics' with id 419c5ac2-e8cb-4115-8aa1-27d41ba0a08e
# Variables
var spec_components_0_properties_requests_memory integer >= 1024, <= 8192;
var spec_components_0_traits_0_properties_replicas integer >= 1, <= 8;
var spec_components_0_properties_requests_cpu integer >= 1, <= 8;
# Metrics. Note that we only emit metrics that are in use. Values will be provided by the solver.
## Raw metrics
param raw_B; # raw_B
## Composite metrics
param comp_C; # comp_C
# Constants
param currentReplicas;
# Performance indicators = composite metrics that have at least one variable in their formula
var pi_F;
var pi_D;
# Performance indicator formulas
subject to define_pi_F : pi_F = pi_D + comp_C;
subject to define_pi_D : pi_D = spec_components_0_traits_0_properties_replicas + comp_C + raw_B;
# TBD: cost parameters - for all components! and use of node-candidates tensor
# Utility functions
maximize firstfunction :
(comp_C*currentReplicas)/spec_components_0_traits_0_properties_replicas;
# Default utility function: specified in message to solver
# Constraints. For constraints we don't have name from GUI, must be created

1
utility-evaluator/src/test/java/resources/Robertsmessage.json Normal file
View File

@ -0,0 +1 @@
{"title":"dummy-app-202403200917","uuid":"fb7f01c6-711e-4a82-96b5-829d62be992a","status":"deploying","content":"apiVersion: \"core.oam.dev/v1beta1\"\r\nkind: \"Application\"\r\nmetadata:\r\n name: \"dummy-app-deploy\"\r\nspec:\r\n components:\r\n - name: \"dummy-app-worker\"\r\n type: \"webservice\"\r\n properties:\r\n image: \"docker.io/rsprat/mytestrepo:v1\"\r\n imagePullPolicy: \"Always\"\r\n cmd:\r\n - \"python\"\r\n - \"worker.py\"\r\n env:\r\n - name: \"mqtt_ip\"\r\n value: \"broker.hivemq.com\"\r\n - name: \"mqtt_port\"\r\n value: \"1883\"\r\n - name: \"mqtt_subscribe_topic\"\r\n value: \"$share/workers/neb/test/input\" \r\n - name: \"nebulous_ems_ip\"\r\n valueFrom:\r\n fieldRef:\r\n fieldPath: status.hostIP\r\n - name: \"nebulous_ems_port\"\r\n value: \"61610\"\r\n - name: \"nebulous_ems_user\"\r\n value: \"aaa\"\r\n - name: \"nebulous_ems_password\"\r\n value: \"111\"\r\n - name: \"nebulous_ems_metrics_topic\"\r\n value: \"realtime.job_process_time_instance\"\r\n traits:\r\n - type: \"scaler\"\r\n properties:\r\n replicas: 1\r\n\r\n\r\n policies:\r\n - name: \"target-default\"\r\n type: \"topology\"\r\n properties:\r\n namespace: \"default\"\r\n workflow:\r\n steps:\r\n - name: \"deploy2default\"\r\n type: \"deploy\"\r\n properties:\r\n policies:\r\n - \"target-default\"","variables":[{"key":"spec_components_0_traits_0_properties_replicas","path":"/spec/components/0/traits/0/properties/replicas","type":"float","meaning":"replicas","value":{"lower_bound":1,"higher_bound":8}}],"environmentVariables":[],"resources":[{"uuid":"408bebd8-2e2a-438b-816a-905ac7fe5280","title":"hello world","platform":"AWS","enabled":true}],"templates":[],"parameters":[],"metrics":[{"type":"raw","name":"MeanJobProcessingLatency","sensor":"MeanJobProcessingLatency","config":[],"inputRaw":{"type":"all","interval":30,"unit":"sec"},"isWindowOutputRaw":true,"outputRaw":{"type":"all","interval":30,"unit":"ms"}}],"sloViolations":{"nodeKey":"db8b6ee3-0aec-4ce3-a9ca-cb3c265a8b8d","isComposite":true,"condition":"AND","not":false,"children":[{"nodeKey":"ca14bada-9c8d-4a96-a5ae-841bfd5ecd38","isComposite":false,"metricName":"MeanJobProcessingLatency","operator":">","value":50}]},"utilityFunctions":[{"name":"f","type":"maximize","expression":{"formula":"spec_components_0_traits_0_properties_replicas","variables":[{"name":"SPEC_COMPONENTS_0_TRAITS_0_PROPERTIES_REPLICAS","value":"spec_components_0_traits_0_properties_replicas"}]}}],"_create":true,"_delete":true}

239
utility-evaluator/src/test/java/resources/app-creation-message-complex (3).json Normal file
View File

File diff suppressed because one or more lines are too long

151
utility-evaluator/src/test/java/resources/app-creation-message-mercabarna.json Normal file
View File

@ -0,0 +1,151 @@
{title=Mercabana Intralogistics, uuid=419c5ac2-e8cb-4115-8aa1-27d41ba0a08e, content=apiVersion: core.oam.dev/v1beta1
kind: Application
metadata:
name: tta-models-v2
namespace: tta-models
spec:
components:
- name: minio-pvc
type: k8s-objects
properties:
objects:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: minio-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 8Gi
cpu: 2
- name: minio
type: webservice
properties:
image: minio/minio
args:
- server
- /data
- --console-address
- ":9090"
exposeType: NodePort
ports:
- name: minio-ui
expose: true
port: 9090
nodePort: 30002
protocol: TCP
- name: minio-api
expose: true
port: 9000
nodePort: 30001
protocol: TCP
env:
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secrets
key: MINIO_ACCESS_KEY
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-secrets
key: MINIO_SECRET_KEY
volumeMounts:
pvc:
- name: minio-storage
mountPath: /data
claimName: minio-pvc
- name: object-detection
type: webservice
properties:
exposeType: NodePort
image: registry.gitlab.com/tta-nebulous/tta-modules/object_detection:minio
imagePullSecrets:
- gitlab-secret
ports:
- name: obj-pred
expose: true
port: 9000
nodePort: 30010
protocol: TCP
env:
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secrets
key: MINIO_ACCESS_KEY
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-secrets
key: MINIO_SECRET_KEY
volumeMounts:
pvc:
- name: minio-storage
mountPath: /minio_data
claimName: minio-pvc
- name: damage-classification
type: webservice
properties:
exposeType: NodePort
image: registry.gitlab.com/tta-nebulous/tta-modules/damage_classification
imagePullSecrets:
- gitlab-secret
ports:
- name: class-pred
expose: true
port: 9001
nodePort: 30011
protocol: TCP
env:
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secrets
key: MINIO_ACCESS_KEY
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-secrets
key: MINIO_SECRET_KEY
volumeMounts:
pvc:
- name: minio-storage
mountPath: /minio_data
claimName: minio-pvc
- name: severity-assessment
type: webservice
properties:
exposeType: NodePort
image: registry.gitlab.com/tta-nebulous/tta-modules/severity_assessment
imagePullSecrets:
- gitlab-secret
ports:
- name: sev-pred
expose: true
port: 9002
nodePort: 30012
protocol: TCP
env:
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secrets
key: MINIO_ACCESS_KEY
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-secrets
key: MINIO_SECRET_KEY
volumeMounts:
pvc:
- name: minio-storage
mountPath: /minio_data
claimName: minio-pvc, variables=[{key=spec.components.properties.objects.spec.resources.requests.cpu, path=/spec.components.properties.objects.spec.resources.requests.cpu, type=float, meaning=spec.components.properties.objects.spec.resources.requests.cpu, value={lower_bound=2, higher_bound=8}}, {key=spec.components.properties.objects.spec.resources.requests.storage, path=/spec.components.properties.objects.spec.resources.requests.storage, type=float, meaning=spec.components.properties.objects.spec.resources.requests.storage, value={lower_bound=10, higher_bound=16}}], resources=[{_id=clrhjsa6j000r0oynb8776rc1, metaType=arrayItem, scopedArrayName=doc.application.resources, uuid=9e6644b6-cb58-4fc5-b5cf-bf7a571d92fe, title=AWs my resource, platform=aws, enabled=true, _docId=clrhjsa6k00170oyn4hghaauk:en:published, _edit=true}], templates=[{_id=cltcwpi2l004t2e5z3kn4isg4, metaType=arrayItem, scopedArrayName=doc.application.templates, id=myid, type=int, minValue=2, maxValue=4, unit=ms, _docId=clrhjsa6k00170oyn4hghaauk:en:published, _edit=true}], parameters=[], metrics=[{_id=clrhjsa6j000s0oyncf9za40p, metaType=arrayItem, scopedArrayName=doc.application.metrics, type=composite, level=Global, components=[], name=cars_by_second, formula=number_of_cars_discovered / total_number_of_cars, isWindowInput=true, input={_id=clrhjsa6j000t0oyngrl8a83d, type=all, interval=0, unit=ms, metaType=object, scopedObjectName=doc.application.input}, isWindowOutput=true, output={_id=clrhjsa6j000u0oyn8enn9mbk, type=all, interval=0, unit=ms, metaType=object, scopedObjectName=doc.application.output}, sensor=, config=[], isWindowInputRaw=false, inputRaw={_id=cltcwla8n001b0lv64v0e49jz, type=null, interval=null, unit=null, metaType=object, scopedObjectName=doc.application.inputRaw}, isWindowOutputRaw=false, outputRaw={_id=clrhjsa6j000v0oyn0waq09do, type=null, interval=null, unit=null, metaType=object, scopedObjectName=doc.application.outputRaw}, _docId=clrhjsa6k00170oyn4hghaauk:en:published, _edit=true, arguments=[number_of_cars_discovered, total_number_of_cars]}, {_id=clrhjsa6j000w0oyn23sychyv, metaType=arrayItem, scopedArrayName=doc.application.metrics, type=raw, level=Global, components=[], name=latency, formula=, isWindowInput=false, input={_id=clrhjsa6j000x0oyn8xb78mv4, type=null, interval=null, unit=null, metaType=object, scopedObjectName=doc.application.input}, isWindowOutput=false, output={_id=clrhjsa6j000y0oyn0f94aiov, type=all, interval=0, unit=ms, metaType=object, scopedObjectName=doc.application.output}, sensor=latency_discovery, config=[{_id=clrhjsa6j000z0oyn0czm7k3e, metaType=arrayItem, scopedArrayName=doc.application.config, name=buffer, value=10, _docId=clrhjsa6k00170oyn4hghaauk:en:published, _edit=true}], isWindowInputRaw=false, inputRaw={_id=cltcwla8n001c0lv6gj4r6rwb, type=null, interval=null, unit=null, metaType=object, scopedObjectName=doc.application.inputRaw}, isWindowOutputRaw=true, outputRaw={_id=clrhjsa6j00100oynhyhs9rdf, type=all, interval=3, unit=ms, metaType=object, scopedObjectName=doc.application.outputRaw}, _docId=clrhjsa6k00170oyn4hghaauk:en:published, _edit=true}], sloViolations={nodeKey=d760437d-81cd-4675-9e40-e8d0ea4b9b4d, isComposite=true, condition=OR, not=false, children=[{nodeKey=b50b961b-6e9a-4f90-842c-e92fbab43f40, isComposite=false, metricName=latency, operator=>, value=10}, {nodeKey=7a91f324-7094-4536-a9b5-1eece0f19d66, isComposite=false, metricName=cars_by_second, operator=<=, value=50}]}, utilityFunctions=[{name=Utility Function 1, type=maximize, expression={formula=A * B + SIN(C), variables=[{name=A, value=cars_by_second}, {name=B, value=spec.components.properties.objects.spec.resources.requests.cpu}, {name=C, value=latency}]}}, {name=Utility Function B, type=constant, expression={formula=D /100, variables=[{name=D, value=Utility_Function_1}]}}], environmentVariables=[], status=deploying, _create=true, _delete=true}

31
utility-evaluator/src/test/java/resources/complex-example-ampl (2).mod Normal file
View File

@ -0,0 +1,31 @@
# AMPL file for application 'Mercabana Intralogistics' with id 419c5ac2-e8cb-4115-8aa1-27d41ba0a08e
# Variables
var spec_components_0_traits_0_properties_replicas >= 1.0, <= 8.0;
# Metrics. Note that we only emit metrics that are in use. Values will be provided by the solver.
## Raw metrics
param raw_B; # raw_B
## Composite metrics
param comp_C;
## Constrants
param currentReplicas;
# Performance indicators = composite metrics that have at least one variable in their formula. Firstly defined:
var pi_F;
var pi_D;
#Performance indicators formulas:
subject to define_pi_F : pi_F = pi_D + comp_C;
subject to define_pi_D : pi_D = spec_components_0_traits_0_properties_replicas + comp_C + raw_B;
# TBD: cost parameters - for all components! and use of node-candidates tensor
# Utility functions
maximize firstfunction :
(comp_C*currentReplicas)/spec_components_0_traits_0_properties_replicas;
# Default utility function: specified in message to solver
# Constraints that include variables / performance indicators

67
utility-evaluator/src/test/java/resources/complex-metric-model.yaml Normal file
View File

@ -0,0 +1,67 @@
apiVersion: nebulous/v1
kind: MetricModel
metadata:
name: 125bff35-7012-4bf2-bf09-cefca295bf73
labels:
app: App-test
templates:
- id: temp
type: int
range:
- 10
- 10000
unit: ms
spec:
components:
- name: spec-comp
metrics: []
scopes:
- name: app-wide-scope
components: []
metrics:
- name: comp_c
type: composite
template: temp
formula: mean(raw_A)
window:
type: sliding
size: 30 sec
output: first 30 sec
- name: pi_D
type: composite
template: temp
formula: spec_components_1_properties_memory + comp_C + raw_B
window:
type: sliding
size: 20 sec
output: first 20 sec
- name: comp_E
type: composite
template: temp
formula: count(raw_A)+comp_C
window:
type: sliding
size: 30 ms
output: all 30 ms
- name: pi_F
type: composite
template: temp
formula: pi_D+comp_C
window:
type: sliding
size: 100 sec
output: last 100 sec
- name: raw_A
type: raw
sensor:
type: raw-a-sensor
config: {}
- name: raw_B
type: raw
sensor:
type: raw-b-sensor
config: {}
requirements:
- name: Combined SLO
type: slo
constraint: "(comp_E < 10)"

43
utility-evaluator/src/test/java/resources/metric-model.yaml Normal file
View File

@ -0,0 +1,43 @@
apiVersion: nebulous/v1
kind: MetricModel
# Optional. Currently, only 'metadata.name' is used (if present)
metadata:
name: dummy-app-deploy
labels:
app: nebulous-dummy-app
common:
time_template: &time_template
id: 'time'
type: double
range: [0, 300]
unit: 'seconds'
latency_template: &latency_template
id: 'time'
type: double
range: [0, 300]
unit: 'seconds'
spec:
scopes:
- name: dummy
components:
- name: dummy-app-worker
metrics:
- name: MeanJobProcessingLatency
type: composite
template: *latency_template
formula: 'mean(RawProcessingLatency)'
level: per_instance
window:
type: sliding
size: '1 min'
output:
type: all
schedule: '30 sec'
- name: RawProcessingLatency
template: *latency_template
sensor:
type: 'job_process_time_instance' #This is a sensor provided by the IoT pub-sub mechanism of the NebulOuS IoT data management
requirements:
- name: MeanJobProcessingLatencySLO
type: slo
constraint: 'MeanJobProcessingLatency >= 50'

1
utility-evaluator/src/test/java/resources/response-all-clouds.json Normal file
View File

File diff suppressed because one or more lines are too long